From 0d9002fcfceda5aaa5d14dddd2835a90e41695fb Mon Sep 17 00:00:00 2001 From: Cyrus Gray <144336577+graycyrus@users.noreply.github.com> Date: Thu, 4 Jun 2026 20:04:22 +0530 Subject: [PATCH] feat(memory-sources): default sync sources on with conservative caps, per-source settings & All In (#3304) --- .claude/memory.md | 12 + .../intelligence/MemorySourcesRegistry.tsx | 377 ++++++--- .../intelligence/SourceSettingsPanel.tsx | 224 ++++++ .../__tests__/MemorySourcesRegistry.test.tsx | 322 ++++++++ app/src/lib/i18n/ar.ts | 26 + app/src/lib/i18n/bn.ts | 26 + app/src/lib/i18n/de.ts | 26 + app/src/lib/i18n/en.ts | 26 + app/src/lib/i18n/es.ts | 27 + app/src/lib/i18n/fr.ts | 26 + app/src/lib/i18n/hi.ts | 26 + app/src/lib/i18n/id.ts | 26 + app/src/lib/i18n/it.ts | 27 + app/src/lib/i18n/ko.ts | 26 + app/src/lib/i18n/pl.ts | 27 + app/src/lib/i18n/pt.ts | 26 + app/src/lib/i18n/ru.ts | 28 + app/src/lib/i18n/zh-CN.ts | 26 + app/src/services/memorySourcesService.test.ts | 71 ++ app/src/services/memorySourcesService.ts | 28 +- src/bin/slack_backfill.rs | 4 + src/openhuman/about_app/catalog_data.rs | 19 + src/openhuman/about_app/catalog_tests.rs | 1 + src/openhuman/composio/ops.rs | 6 + src/openhuman/config/schema/types.rs | 13 + src/openhuman/memory_sources/mod.rs | 7 +- src/openhuman/memory_sources/reconcile.rs | 206 +++++ src/openhuman/memory_sources/registry.rs | 164 +++- src/openhuman/memory_sources/rpc.rs | 112 ++- src/openhuman/memory_sources/schemas.rs | 52 +- src/openhuman/memory_sync/composio/bus.rs | 39 +- src/openhuman/memory_sync/composio/mod.rs | 24 + .../memory_sync/composio/periodic.rs | 25 + .../composio/providers/clickup/provider.rs | 84 +- .../composio/providers/github/provider.rs | 114 ++- .../composio/providers/gmail/provider.rs | 71 +- .../memory_sync/composio/providers/helpers.rs | 172 ++++ .../composio/providers/linear/provider.rs | 75 +- .../composio/providers/notion/provider.rs | 77 +- .../composio/providers/slack/provider.rs | 95 ++- .../composio/providers/slack/rpc.rs | 2 + .../memory_sync/composio/providers/types.rs | 20 + src/openhuman/task_sources/ops.rs | 4 + src/openhuman/task_sources/pipeline.rs | 2 + .../config_auth_app_state_connectivity_e2e.rs | 1 + .../memory_sync_providers_raw_coverage_e2e.rs | 755 ++++++++++++++++++ tests/memory_sync_round23_raw_coverage_e2e.rs | 2 + .../memory_sync_slack_bus_raw_coverage_e2e.rs | 2 + tests/memory_sync_sources_raw_coverage_e2e.rs | 2 + ...mory_sync_tree_round21_raw_coverage_e2e.rs | 2 + tests/memory_threads_raw_coverage_e2e.rs | 2 + tests/memory_tree_sync_raw_coverage_e2e.rs | 2 + 52 files changed, 3405 insertions(+), 154 deletions(-) create mode 100644 app/src/components/intelligence/SourceSettingsPanel.tsx create mode 100644 app/src/components/intelligence/__tests__/MemorySourcesRegistry.test.tsx diff --git a/.claude/memory.md b/.claude/memory.md index 4e50acc1b..50ea71a60 100644 --- a/.claude/memory.md +++ b/.claude/memory.md @@ -300,3 +300,15 @@ Quick reference for anyone starting with Claude on this project. Updated by the - **Upstream `main` has 5 Vitest failures and 4 TypeScript compile errors** — Caused by missing iOS experimental dependencies: `@noble/ciphers/chacha`, `@noble/ciphers/webcrypto`, `qrcode.react`, `@tauri-apps/plugin-barcode-scanner`. Breaks `pnpm compile`, `pnpm build`, `pnpm test:coverage` on a clean checkout. Always verify by stashing changes and running checks on the base branch before blaming your PR. - **`cargo fmt` must run after codecrusher** — codecrusher does not reliably produce `cargo fmt`-clean Rust. Always run `cargo fmt --manifest-path Cargo.toml` after codecrusher finishes and before committing. + +## Memory Sync Sources — Defaults ON + Per-Source UI (Issue #3293) + +- **Conservative caps registry** — `composio_defaults_for_toolkit(toolkit) -> (max_items, sync_depth_days)` in `src/openhuman/memory_sources/registry.rs` is the single source of truth. Values: gmail 100/30, slack 50/14, notion 30/30, linear 50/30, clickup 50/30, github 50/30, generic 30/14. Non-Composio defaults (GithubRepo 10PR/10issue/50commit, RSS 20, Twitter 7d) live in `apply_kind_defaults` in `rpc.rs`. +- **`upsert_composio_source` now defaults ON** — Registers `enabled: true` with caps applied from the registry. Previously registered `enabled: false` with no caps. +- **Cap enforcement: 3 construction sites, not 1** — `ProviderContext` (`memory_sync/composio/providers/types.rs`) carries `max_items`/`sync_depth_days`. All three sites must populate caps from the registry entry: `composio/mod.rs run_connection_sync`, `composio/periodic.rs`, `composio/bus.rs`. Each provider reads `ctx.max_items`/`ctx.sync_depth_days` for pagination + date clamping. Shared helpers: `pages_for_max_items` / `epoch_floor_from_depth` in `providers/helpers.rs`. +- **First-sync gotcha on new connections** — `bus.rs on_connection_created` fires BEFORE `upsert_composio_source`, so caps are (None, None) on the brand-new connection's first sync — bounded only by internal `MAX_PAGES`, not registry caps. Documented in code; intentional. +- **`memory_sources_apply_all_in` RPC** — Zero params → `{ sources, sync_triggered }`. Enables all sources, clears caps to None (falls back to internal `MAX_PAGES` ceilings, ~500 items, not truly unlimited), triggers sync per source. +- **Retroactive migration** — `apply_composio_source_caps_migration` in `reconcile.rs`, guarded by `Config.composio_source_caps_migrated: bool` (`#[serde(default)]`), runs once from `ensure_composio_sources`. Only touches Composio entries with `!enabled && max_items.is_none() && sync_depth_days.is_none()` — never overwrites user-customized caps. +- **`MemorySourcePatch` was missing limit fields** — `max_commits`, `max_issues`, `max_prs` were absent from `MemorySourcePatch`, `update_source`, and the update schema. Also missing from the TS `MemorySourceEntry` interface in `memorySourcesService.ts`. Both were fixed in this issue. +- **Per-source settings UI** — `SourceSettingsPanel.tsx` (sibling of `MemorySourcesRegistry.tsx`) with a `KIND_FIELDS` map driving which limit fields appear per source kind. Empty input = omit from patch (use default); number = set. "All In" button uses `ConfirmationModal` (primary-500 prominent). Toasts via existing `onToast` prop. +- **`pnpm i18n:english:check` is pre-failing** — Exits with `total unexpected English: 1312` on a clean base tree. Confirm your new keys aren't among the failures rather than expecting exit 0. `pnpm i18n:check` (key parity) is the real gate. diff --git a/app/src/components/intelligence/MemorySourcesRegistry.tsx b/app/src/components/intelligence/MemorySourcesRegistry.tsx index 988a6e0f0..1d14ca0ef 100644 --- a/app/src/components/intelligence/MemorySourcesRegistry.tsx +++ b/app/src/components/intelligence/MemorySourcesRegistry.tsx @@ -8,10 +8,11 @@ * row dispatches `openhuman.memory_sources_sync` which runs in the * background and emits MemorySyncStageChanged events. */ -import { useCallback, useEffect, useMemo, useState } from 'react'; +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { useT } from '../../lib/i18n/I18nContext'; import { + applyAllIn, type FreshnessLabel, listMemorySources, type MemorySourceEntry, @@ -23,9 +24,14 @@ import { syncMemorySource, updateMemorySource, } from '../../services/memorySourcesService'; -import type { ToastNotification } from '../../types/intelligence'; +import type { + ConfirmationModal as ConfirmationModalType, + ToastNotification, +} from '../../types/intelligence'; import { memoryTreeFlushSource } from '../../utils/tauriCommands/memoryTree'; import { AddMemorySourceDialog } from './AddMemorySourceDialog'; +import { ConfirmationModal } from './ConfirmationModal'; +import { SourceSettingsPanel } from './SourceSettingsPanel'; interface MemorySourcesRegistryProps { onToast?: (toast: Omit) => void; @@ -59,6 +65,10 @@ export function MemorySourcesRegistry({ const [syncingId, setSyncingId] = useState(null); const [buildingId, setBuildingId] = useState(null); const [syncProgress, setSyncProgress] = useState>(new Map()); + const [allInModalOpen, setAllInModalOpen] = useState(false); + const [applyingAllIn, setApplyingAllIn] = useState(false); + const allInInFlightRef = useRef(false); + const [expandedSettingsId, setExpandedSettingsId] = useState(null); useEffect(() => { const handler = (e: Event) => { @@ -223,23 +233,84 @@ export function MemorySourcesRegistry({ [onToast, refresh, t] ); + const handleConfirmAllIn = useCallback(async () => { + if (allInInFlightRef.current) return; + allInInFlightRef.current = true; + setApplyingAllIn(true); + try { + const result = await applyAllIn(); + setSources(result.sources); + onToast?.({ type: 'success', title: t('memorySources.allIn.success') }); + } catch (err) { + onToast?.({ + type: 'error', + title: t('memorySources.allIn.failed'), + message: err instanceof Error ? err.message : String(err), + }); + } finally { + allInInFlightRef.current = false; + setApplyingAllIn(false); + setAllInModalOpen(false); + } + }, [onToast, t]); + + const handleSettingsSaved = useCallback((updated: MemorySourceEntry) => { + setSources(prev => prev.map(s => (s.id === updated.id ? updated : s))); + }, []); + + const handleToggleSettings = useCallback((sourceId: string) => { + setExpandedSettingsId(prev => (prev === sourceId ? null : sourceId)); + }, []); + + const allInModal: ConfirmationModalType = { + isOpen: allInModalOpen, + title: t('memorySources.allIn.title'), + message: t('memorySources.allIn.message'), + confirmText: t('memorySources.allIn.confirm'), + cancelText: t('memorySources.allIn.cancel'), + destructive: false, + onConfirm: () => { + void handleConfirmAllIn(); + }, + onCancel: () => { + setAllInModalOpen(false); + }, + }; + return (
-
+

{t('memorySources.title')}

- +
+ + +
{loading ? ( @@ -256,10 +327,14 @@ export function MemorySourcesRegistry({ isSyncing={syncingId === source.id} isBuilding={buildingId === source.id} progress={syncProgress.get(source.id) ?? null} + settingsExpanded={expandedSettingsId === source.id} onToggle={handleToggle} onRemove={handleRemove} onSync={handleSync} onBuild={handleBuild} + onToggleSettings={handleToggleSettings} + onSettingsSaved={handleSettingsSaved} + onToast={onToast} /> ))} @@ -270,6 +345,10 @@ export function MemorySourcesRegistry({ onClose={() => setDialogOpen(false)} onAdded={handleAdded} /> + + {allInModalOpen && ( + setAllInModalOpen(false)} /> + )}
); } @@ -280,10 +359,14 @@ interface SourceRowProps { isSyncing: boolean; isBuilding: boolean; progress: SyncProgress | null; + settingsExpanded: boolean; onToggle: (source: MemorySourceEntry) => void; onRemove: (source: MemorySourceEntry) => void; onSync: (source: MemorySourceEntry) => void; onBuild: (source: MemorySourceEntry) => void; + onToggleSettings: (sourceId: string) => void; + onSettingsSaved: (updated: MemorySourceEntry) => void; + onToast?: (toast: Omit) => void; } function SourceRow({ @@ -292,10 +375,14 @@ function SourceRow({ isSyncing, isBuilding, progress, + settingsExpanded, onToggle, onRemove, onSync, onBuild, + onToggleSettings, + onSettingsSaved, + onToast, }: SourceRowProps) { const { t } = useT(); const icon = SOURCE_KIND_ICONS[source.kind] ?? '📄'; @@ -304,125 +391,146 @@ function SourceRow({ const lastSync = status ? relativeTimestamp(status.last_chunk_at_ms, t) : null; return ( -
  • -
    -
    - {icon} - - {source.label} - - - {kindLabel} - - {status && status.chunks_synced > 0 && } -
    - {detail && ( -

    - {detail} -

    - )} - {progress && ( -
    -
    - {progress.stage} - {progress.percent !== null && ( - - {progress.percent}% - - )} - {progress.detail && ( - - {progress.detail} - - )} -
    -
    -
    -
    -
    - )} - {!progress && status && (status.chunks_synced > 0 || status.chunks_pending > 0) && ( -
    - - {status.chunks_synced.toLocaleString()} {t('sync.chunks')} +
  • +
    +
    +
    + {icon} + + {source.label} - {lastSync && ( - - {t('sync.lastChunk')} {lastSync} - - )} - {status.chunks_pending > 0 && ( - - {status.chunks_pending.toLocaleString()} {t('sync.pending')} - - )} + + {kindLabel} + + {status && status.chunks_synced > 0 && }
    - )} -
    -
    - + - + - - + + + + +
    + {settingsExpanded && ( + + )}
    • ); } @@ -573,3 +681,38 @@ function Spinner() { ); } + +function GearIcon() { + return ( + + ); +} + +function AllInIcon() { + return ( + + ); +} diff --git a/app/src/components/intelligence/SourceSettingsPanel.tsx b/app/src/components/intelligence/SourceSettingsPanel.tsx new file mode 100644 index 000000000..24eeb9b5e --- /dev/null +++ b/app/src/components/intelligence/SourceSettingsPanel.tsx @@ -0,0 +1,224 @@ +/** + * Expandable per-source sync-settings panel. + * + * Rendered inline below a SourceRow when the gear icon is clicked. + * Shows only the limit fields relevant to the source's kind, seeded from + * the current entry values. Empty input = omit the field on save (meaning + * "use backend default / unlimited"). + * + * On Save calls `updateMemorySource` and notifies the parent via + * `onSaved` with the updated entry. + */ +import { useCallback, useState } from 'react'; + +import { useT } from '../../lib/i18n/I18nContext'; +import { + type MemorySourceEntry, + type SourceKind, + updateMemorySource, +} from '../../services/memorySourcesService'; + +// Which limit fields are relevant per kind. Order determines display order. +// Only caps that are actually enforced at sync time are surfaced — the +// per-sync token/cost budgets are not yet wired into the sync loop, so they +// are intentionally omitted here. +const KIND_FIELDS: Record> = { + composio: ['sync_depth_days', 'max_items'], + github_repo: ['max_prs', 'max_issues', 'max_commits', 'sync_depth_days'], + rss_feed: ['max_items', 'sync_depth_days'], + twitter_query: ['since_days'], + web_page: ['sync_depth_days'], + folder: ['sync_depth_days'], +}; + +// i18n key for each field label +const FIELD_LABEL_KEYS: Record = { + max_prs: 'memorySources.settings.maxPrs', + max_issues: 'memorySources.settings.maxIssues', + max_commits: 'memorySources.settings.maxCommits', + max_items: 'memorySources.settings.maxItems', + since_days: 'memorySources.settings.sinceDays', + sync_depth_days: 'memorySources.settings.syncDepthDays', +}; + +type LimitFields = Pick< + MemorySourceEntry, + 'max_prs' | 'max_issues' | 'max_commits' | 'max_items' | 'since_days' | 'sync_depth_days' +>; + +// Item-count caps where a "Maxed" badge is meaningful (synced count vs cap). +// Time-window (sync_depth_days/since_days) and budget (tokens/cost) caps don't +// map to a chunk count, so they never show "Maxed". +const COUNT_FIELDS = new Set([ + 'max_items', + 'max_prs', + 'max_issues', + 'max_commits', +]); + +interface SourceSettingsPanelProps { + source: MemorySourceEntry; + /** Chunks already synced for this source — drives the "Maxed" badge. */ + syncedCount?: number; + onSaved: (updated: MemorySourceEntry) => void; + onToast?: (toast: { type: 'success' | 'error'; title: string; message?: string }) => void; +} + +export function SourceSettingsPanel({ + source, + syncedCount, + onSaved, + onToast, +}: SourceSettingsPanelProps) { + const { t } = useT(); + const fields = KIND_FIELDS[source.kind] ?? []; + + // Hold each field as a string so inputs can be freely edited. Seeded from the + // entry's stored cap (fetched from the backend — the single source of truth; + // the caps migration writes conservative defaults onto the entry). An empty + // field genuinely means "no cap / unlimited". + const [values, setValues] = useState>(() => { + const init: Record = {}; + for (const f of fields) { + const stored = source[f as keyof MemorySourceEntry]; + init[f] = stored != null ? String(stored) : ''; + } + return init; + }); + + const [saving, setSaving] = useState(false); + + const handleChange = useCallback((field: string, value: string) => { + setValues(prev => ({ ...prev, [field]: value })); + }, []); + + const handleSave = useCallback(async () => { + setSaving(true); + try { + const patch: Partial = {}; + for (const f of fields) { + const raw = values[f]; + if (raw !== '' && raw !== undefined) { + const parsed = Number(raw); + if (!Number.isFinite(parsed) || !Number.isInteger(parsed) || parsed < 0) { + onToast?.({ type: 'error', title: t('memorySources.settings.saveFailed') }); + return; + } + (patch as Record)[f] = parsed; + } + // Empty string → omit from patch (backend treats absence as "default") + } + const updated = await updateMemorySource(source.id, patch); + onSaved(updated); + onToast?.({ type: 'success', title: t('memorySources.settings.saved') }); + } catch (err) { + onToast?.({ + type: 'error', + title: t('memorySources.settings.saveFailed'), + message: err instanceof Error ? err.message : String(err), + }); + } finally { + setSaving(false); + } + }, [fields, source.id, values, onSaved, onToast, t]); + + if (fields.length === 0) return null; + + // Display name for the tooltip — the toolkit slug (title-cased) for Composio + // sources, else the source label. + const toolkitName = source.toolkit + ? source.toolkit.charAt(0).toUpperCase() + source.toolkit.slice(1) + : source.label; + const unlimitedTooltip = t('memorySources.settings.unlimitedTooltip').replace( + '{toolkit}', + toolkitName + ); + + return ( +
    +

    + {t('memorySources.settings.title')} +

    +
    + {fields.map(field => { + const cap = Number(values[field]); + const isUnlimited = (values[field] ?? '') === ''; + const isMaxed = + COUNT_FIELDS.has(field) && + !isUnlimited && + Number.isFinite(cap) && + typeof syncedCount === 'number' && + syncedCount >= cap; + return ( +
    + + handleChange(field, e.target.value)} + placeholder={t('memorySources.settings.unlimited')} + className="w-full rounded-md border border-stone-200 bg-white px-2.5 py-1.5 text-xs font-mono + text-stone-800 placeholder:text-stone-400 + dark:border-neutral-600 dark:bg-neutral-900 dark:text-neutral-200 + dark:placeholder:text-neutral-500 + focus:outline-none focus:ring-2 focus:ring-primary-200" + /> +
    + ); + })} +
    +
    + +
    +
    + ); +} + +function InfoIcon() { + return ( + + ); +} diff --git a/app/src/components/intelligence/__tests__/MemorySourcesRegistry.test.tsx b/app/src/components/intelligence/__tests__/MemorySourcesRegistry.test.tsx new file mode 100644 index 000000000..bc8c32d60 --- /dev/null +++ b/app/src/components/intelligence/__tests__/MemorySourcesRegistry.test.tsx @@ -0,0 +1,322 @@ +/** + * Unit tests for MemorySourcesRegistry — All In button, gear/settings panel, + * per-kind field visibility, Save, and existing toggle behaviour. + */ +import { fireEvent, screen, waitFor, within } from '@testing-library/react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import * as service from '../../../services/memorySourcesService'; +import type { MemorySourceEntry } from '../../../services/memorySourcesService'; +import { renderWithProviders } from '../../../test/test-utils'; +import { MemorySourcesRegistry } from '../MemorySourcesRegistry'; + +// Mock the entire service so we don't hit RPC +vi.mock('../../../services/memorySourcesService', async () => { + const actual = await vi.importActual( + '../../../services/memorySourcesService' + ); + return { + ...actual, + listMemorySources: vi.fn(), + memorySourcesStatusList: vi.fn(), + updateMemorySource: vi.fn(), + removeMemorySource: vi.fn(), + syncMemorySource: vi.fn(), + applyAllIn: vi.fn(), + }; +}); + +// Mock tauriCommands/memoryTree — not needed in these tests +vi.mock('../../../utils/tauriCommands/memoryTree', () => ({ + memoryTreeFlushSource: vi.fn().mockResolvedValue({ seals_fired: 0 }), +})); + +const mockedList = vi.mocked(service.listMemorySources); +const mockedStatus = vi.mocked(service.memorySourcesStatusList); +const mockedUpdate = vi.mocked(service.updateMemorySource); +const mockedApplyAllIn = vi.mocked(service.applyAllIn); + +function makeSource(overrides: Partial = {}): MemorySourceEntry { + return { + id: 'src_1', + kind: 'github_repo', + label: 'My Repo', + enabled: true, + url: 'https://github.com/org/repo', + ...overrides, + }; +} + +function setup(sources: MemorySourceEntry[] = [makeSource()]) { + mockedList.mockResolvedValue(sources); + mockedStatus.mockResolvedValue([]); + const onToast = vi.fn(); + const result = renderWithProviders( + , + {} + ); + return { ...result, onToast }; +} + +describe('MemorySourcesRegistry', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + // ------------------------------------------------------------------------- + // Basic render + // ------------------------------------------------------------------------- + it('renders loaded sources list', async () => { + setup([makeSource({ label: 'Work Repo' })]); + await screen.findByText('Work Repo'); + expect(screen.getByTestId('memory-sources')).toBeInTheDocument(); + }); + + it('renders empty state when no sources', async () => { + mockedList.mockResolvedValue([]); + mockedStatus.mockResolvedValue([]); + renderWithProviders(); + await screen.findByText(/no memory sources/i); + }); + + // ------------------------------------------------------------------------- + // Toggle (existing behaviour) + // ------------------------------------------------------------------------- + it('toggle calls updateMemorySource and flips state', async () => { + const source = makeSource({ enabled: true }); + mockedUpdate.mockResolvedValue({ ...source, enabled: false }); + setup([source]); + await screen.findByText('My Repo'); + + const toggle = screen.getByTitle(/disable/i); + fireEvent.click(toggle); + + await waitFor(() => { + expect(mockedUpdate).toHaveBeenCalledWith('src_1', { enabled: false }); + }); + }); + + // ------------------------------------------------------------------------- + // All In button + // ------------------------------------------------------------------------- + it('All In button is rendered in the header', async () => { + setup(); + await screen.findByText('My Repo'); + expect(screen.getByTestId('all-in-button')).toBeInTheDocument(); + }); + + it('clicking All In opens a confirmation modal', async () => { + setup(); + await screen.findByText('My Repo'); + + fireEvent.click(screen.getByTestId('all-in-button')); + + // The modal should appear + await screen.findByText('Go All In?'); + expect( + screen.getByText(/This enables every memory source and removes all sync limits/i) + ).toBeInTheDocument(); + }); + + it('cancelling All In modal closes it without calling applyAllIn', async () => { + setup(); + await screen.findByText('My Repo'); + + fireEvent.click(screen.getByTestId('all-in-button')); + await screen.findByText('Go All In?'); + + // Click the No / cancel button + fireEvent.click(screen.getByText('No')); + + await waitFor(() => { + expect(screen.queryByText('Go All In?')).not.toBeInTheDocument(); + }); + expect(mockedApplyAllIn).not.toHaveBeenCalled(); + }); + + it('confirming All In calls applyAllIn, updates sources, and shows success toast', async () => { + const updatedSrc = makeSource({ id: 'src_2', label: 'New Repo', enabled: true }); + mockedApplyAllIn.mockResolvedValue({ sources: [updatedSrc], sync_triggered: 1 }); + + const { onToast } = setup(); + await screen.findByText('My Repo'); + + fireEvent.click(screen.getByTestId('all-in-button')); + await screen.findByText('Go All In?'); + + fireEvent.click(screen.getByText('Yes')); + + await waitFor(() => { + expect(mockedApplyAllIn).toHaveBeenCalledOnce(); + }); + + await waitFor(() => { + expect(onToast).toHaveBeenCalledWith(expect.objectContaining({ type: 'success' })); + }); + + // Modal should close + await waitFor(() => { + expect(screen.queryByText('Go All In?')).not.toBeInTheDocument(); + }); + }); + + it('All In failure shows error toast', async () => { + mockedApplyAllIn.mockRejectedValue(new Error('RPC error')); + + const { onToast } = setup(); + await screen.findByText('My Repo'); + + fireEvent.click(screen.getByTestId('all-in-button')); + await screen.findByText('Go All In?'); + + fireEvent.click(screen.getByText('Yes')); + + await waitFor(() => { + expect(onToast).toHaveBeenCalledWith(expect.objectContaining({ type: 'error' })); + }); + }); + + // ------------------------------------------------------------------------- + // Gear / settings panel — toggling + // ------------------------------------------------------------------------- + it('gear button renders for each source row', async () => { + setup([makeSource({ id: 'src_1' }), makeSource({ id: 'src_2', label: 'Second' })]); + await screen.findByText('My Repo'); + + expect(screen.getByTestId('memory-source-settings-src_1')).toBeInTheDocument(); + expect(screen.getByTestId('memory-source-settings-src_2')).toBeInTheDocument(); + }); + + it('clicking gear expands the settings panel for that source', async () => { + setup([makeSource({ id: 'src_1', kind: 'github_repo' })]); + await screen.findByText('My Repo'); + + expect(screen.queryByTestId('source-settings-panel-src_1')).not.toBeInTheDocument(); + + fireEvent.click(screen.getByTestId('memory-source-settings-src_1')); + + expect(screen.getByTestId('source-settings-panel-src_1')).toBeInTheDocument(); + }); + + it('clicking gear again collapses the settings panel', async () => { + setup([makeSource({ id: 'src_1', kind: 'github_repo' })]); + await screen.findByText('My Repo'); + + const gearBtn = screen.getByTestId('memory-source-settings-src_1'); + fireEvent.click(gearBtn); + expect(screen.getByTestId('source-settings-panel-src_1')).toBeInTheDocument(); + + fireEvent.click(gearBtn); + expect(screen.queryByTestId('source-settings-panel-src_1')).not.toBeInTheDocument(); + }); + + // ------------------------------------------------------------------------- + // Settings panel — field visibility per kind + // ------------------------------------------------------------------------- + it('github_repo settings panel shows max_prs, max_issues, max_commits, sync_depth_days', async () => { + setup([makeSource({ id: 'src_1', kind: 'github_repo' })]); + await screen.findByText('My Repo'); + + fireEvent.click(screen.getByTestId('memory-source-settings-src_1')); + + const panel = screen.getByTestId('source-settings-panel-src_1'); + expect(within(panel).getByLabelText(/max pull requests/i)).toBeInTheDocument(); + expect(within(panel).getByLabelText(/max issues/i)).toBeInTheDocument(); + expect(within(panel).getByLabelText(/max commits/i)).toBeInTheDocument(); + expect(within(panel).getByLabelText(/sync depth/i)).toBeInTheDocument(); + }); + + it('composio settings panel shows sync_depth_days and max_items but NOT max_prs', async () => { + setup([makeSource({ id: 'src_1', kind: 'composio', toolkit: 'github' })]); + await screen.findByText('My Repo'); + + fireEvent.click(screen.getByTestId('memory-source-settings-src_1')); + + const panel = screen.getByTestId('source-settings-panel-src_1'); + expect(within(panel).getByLabelText(/max items/i)).toBeInTheDocument(); + expect(within(panel).getByLabelText(/sync depth/i)).toBeInTheDocument(); + expect(within(panel).queryByLabelText(/max pull requests/i)).not.toBeInTheDocument(); + }); + + it('rss_feed settings panel shows max_items and sync_depth_days but NOT max_prs', async () => { + setup([makeSource({ id: 'src_1', kind: 'rss_feed', url: 'https://example.com/feed.xml' })]); + await screen.findByText('My Repo'); + + fireEvent.click(screen.getByTestId('memory-source-settings-src_1')); + + const panel = screen.getByTestId('source-settings-panel-src_1'); + expect(within(panel).getByLabelText(/max items/i)).toBeInTheDocument(); + expect(within(panel).getByLabelText(/sync depth/i)).toBeInTheDocument(); + expect(within(panel).queryByLabelText(/max pull requests/i)).not.toBeInTheDocument(); + expect(within(panel).queryByLabelText(/max commits/i)).not.toBeInTheDocument(); + }); + + // ------------------------------------------------------------------------- + // Settings panel — Save + // ------------------------------------------------------------------------- + it('Save in settings panel calls updateMemorySource with numeric patch', async () => { + const source = makeSource({ id: 'src_1', kind: 'github_repo' }); + const updated = { ...source, max_prs: 50 }; + mockedUpdate.mockResolvedValue(updated); + + const { onToast } = setup([source]); + await screen.findByText('My Repo'); + + fireEvent.click(screen.getByTestId('memory-source-settings-src_1')); + + const panel = screen.getByTestId('source-settings-panel-src_1'); + const maxPrsInput = within(panel).getByLabelText(/max pull requests/i); + fireEvent.change(maxPrsInput, { target: { value: '50' } }); + + const saveBtn = within(panel).getByText('Save'); + fireEvent.click(saveBtn); + + await waitFor(() => { + expect(mockedUpdate).toHaveBeenCalledWith('src_1', expect.objectContaining({ max_prs: 50 })); + }); + + await waitFor(() => { + expect(onToast).toHaveBeenCalledWith(expect.objectContaining({ type: 'success' })); + }); + }); + + it('empty input is omitted from the save patch (not sent as 0)', async () => { + const source = makeSource({ id: 'src_1', kind: 'github_repo', max_prs: 10 }); + mockedUpdate.mockResolvedValue(source); + + setup([source]); + await screen.findByText('My Repo'); + + fireEvent.click(screen.getByTestId('memory-source-settings-src_1')); + + const panel = screen.getByTestId('source-settings-panel-src_1'); + const maxPrsInput = within(panel).getByLabelText(/max pull requests/i); + + // Clear the field + fireEvent.change(maxPrsInput, { target: { value: '' } }); + + fireEvent.click(within(panel).getByText('Save')); + + await waitFor(() => { + expect(mockedUpdate).toHaveBeenCalledWith( + 'src_1', + expect.not.objectContaining({ max_prs: expect.anything() }) + ); + }); + }); + + it('Save failure shows error toast', async () => { + mockedUpdate.mockRejectedValue(new Error('Save failed')); + + const { onToast } = setup([makeSource({ kind: 'github_repo' })]); + await screen.findByText('My Repo'); + + fireEvent.click(screen.getByTestId('memory-source-settings-src_1')); + const panel = screen.getByTestId('source-settings-panel-src_1'); + fireEvent.click(within(panel).getByText('Save')); + + await waitFor(() => { + expect(onToast).toHaveBeenCalledWith(expect.objectContaining({ type: 'error' })); + }); + }); +}); diff --git a/app/src/lib/i18n/ar.ts b/app/src/lib/i18n/ar.ts index b11e349a9..1eaa48ca1 100644 --- a/app/src/lib/i18n/ar.ts +++ b/app/src/lib/i18n/ar.ts @@ -1982,6 +1982,32 @@ const messages: TranslationMap = { 'memorySources.build.successTitle': 'تم بناء الشجرة', 'memorySources.build.failedTitle': 'فشل البناء', 'memorySources.build.sealsMessage': 'تم إتمام عملية الختم', + 'memorySources.allIn.button': 'الكل', + 'memorySources.allIn.title': 'هل تريد تفعيل الكل؟', + 'memorySources.allIn.message': + 'سيؤدي هذا إلى تمكين جميع مصادر الذاكرة وإزالة جميع قيود المزامنة. يبني أغنى رسم بياني للذاكرة، لكنه قد يستهلك المزيد من الاعتمادات.', + 'memorySources.allIn.confirm': 'نعم', + 'memorySources.allIn.cancel': 'لا', + 'memorySources.allIn.success': 'تم تمكين جميع المصادر بدون قيود. بدأت المزامنة.', + 'memorySources.allIn.failed': 'تعذّر تطبيق خيار الكل. يرجى المحاولة مرة أخرى.', + 'memorySources.settings.button': 'الإعدادات', + 'memorySources.settings.title': 'إعدادات المزامنة', + 'memorySources.settings.maxPrs': 'أقصى عدد لطلبات السحب', + 'memorySources.settings.maxIssues': 'أقصى عدد للمشكلات', + 'memorySources.settings.maxCommits': 'أقصى عدد للإيداعات', + 'memorySources.settings.maxItems': 'أقصى عدد للعناصر', + 'memorySources.settings.sinceDays': 'فترة البحث (بالأيام)', + 'memorySources.settings.syncDepthDays': 'عمق المزامنة (بالأيام)', + 'memorySources.settings.maxTokens': 'أقصى عدد للرموز لكل مزامنة', + 'memorySources.settings.maxCost': 'الحد الأقصى للتكلفة لكل مزامنة (دولار)', + 'memorySources.settings.unlimited': 'غير محدود', + 'memorySources.settings.unlimitedTooltip': + 'لقد اخترت مزامنة الحد الأقصى لـ {toolkit}. يمكنك تغيير الحدود من هنا.', + 'memorySources.settings.maxed': 'مكتمل', + 'memorySources.settings.save': 'حفظ', + 'memorySources.settings.saving': 'جارٍ الحفظ…', + 'memorySources.settings.saved': 'تم حفظ الإعدادات', + 'memorySources.settings.saveFailed': 'تعذّر حفظ الإعدادات', 'backend.aiBackend': 'خلفية الذكاء الاصطناعي', 'backend.cloud': 'سحابي', 'backend.recommended': 'موصى به', diff --git a/app/src/lib/i18n/bn.ts b/app/src/lib/i18n/bn.ts index 5a6e782ab..df159b3bc 100644 --- a/app/src/lib/i18n/bn.ts +++ b/app/src/lib/i18n/bn.ts @@ -2021,6 +2021,32 @@ const messages: TranslationMap = { 'memorySources.build.successTitle': 'ট্রি তৈরি হয়েছে', 'memorySources.build.failedTitle': 'তৈরি ব্যর্থ', 'memorySources.build.sealsMessage': 'সিল সম্পন্ন হয়েছে', + 'memorySources.allIn.button': 'সব চালু', + 'memorySources.allIn.title': 'সব সক্রিয় করবেন?', + 'memorySources.allIn.message': + 'এটি সমস্ত মেমরি উৎস সক্রিয় করবে এবং সিঙ্ক সীমা সরিয়ে দেবে। এটি সমৃদ্ধতম মেমরি গ্রাফ তৈরি করে, তবে আরও ক্রেডিট ব্যবহার করতে পারে।', + 'memorySources.allIn.confirm': 'হ্যাঁ', + 'memorySources.allIn.cancel': 'না', + 'memorySources.allIn.success': 'সব উৎস সীমা ছাড়া সক্রিয় হয়েছে। সিঙ্ক শুরু হয়েছে।', + 'memorySources.allIn.failed': 'সব চালু করা সম্ভব হয়নি। আবার চেষ্টা করুন।', + 'memorySources.settings.button': 'সেটিংস', + 'memorySources.settings.title': 'সিঙ্ক সেটিংস', + 'memorySources.settings.maxPrs': 'সর্বোচ্চ পুল রিকোয়েস্ট', + 'memorySources.settings.maxIssues': 'সর্বোচ্চ ইস্যু', + 'memorySources.settings.maxCommits': 'সর্বোচ্চ কমিট', + 'memorySources.settings.maxItems': 'সর্বোচ্চ আইটেম', + 'memorySources.settings.sinceDays': 'পূর্ববর্তী সময় (দিনে)', + 'memorySources.settings.syncDepthDays': 'সিঙ্ক গভীরতা (দিনে)', + 'memorySources.settings.maxTokens': 'প্রতি সিঙ্কে সর্বোচ্চ টোকেন', + 'memorySources.settings.maxCost': 'প্রতি সিঙ্কে সর্বোচ্চ খরচ (USD)', + 'memorySources.settings.unlimited': 'সীমাহীন', + 'memorySources.settings.unlimitedTooltip': + 'আপনি {toolkit}-এর জন্য সর্বাধিক সিঙ্ক করার অপশন বেছে নিয়েছেন। আপনি এখানে সীমা পরিবর্তন করতে পারেন।', + 'memorySources.settings.maxed': 'পূর্ণ', + 'memorySources.settings.save': 'সংরক্ষণ', + 'memorySources.settings.saving': 'সংরক্ষণ হচ্ছে…', + 'memorySources.settings.saved': 'সেটিংস সংরক্ষিত হয়েছে', + 'memorySources.settings.saveFailed': 'সেটিংস সংরক্ষণ করা সম্ভব হয়নি', 'backend.aiBackend': 'AI ব্যাকএন্ড', 'backend.cloud': 'ক্লাউড', 'backend.recommended': 'প্রস্তাবিত', diff --git a/app/src/lib/i18n/de.ts b/app/src/lib/i18n/de.ts index 1f602f114..9bfe94940 100644 --- a/app/src/lib/i18n/de.ts +++ b/app/src/lib/i18n/de.ts @@ -2072,6 +2072,32 @@ const messages: TranslationMap = { 'memorySources.build.successTitle': 'Baum erstellt', 'memorySources.build.failedTitle': 'Erstellung fehlgeschlagen', 'memorySources.build.sealsMessage': 'Versiegelung(en) abgeschlossen', + 'memorySources.allIn.button': 'Alles aktivieren', + 'memorySources.allIn.title': 'Alles aktivieren?', + 'memorySources.allIn.message': + 'Dadurch werden alle Speicherquellen aktiviert und alle Synchronisierungslimits entfernt. Es erstellt den reichhaltigsten Speichergraphen, kann aber mehr Credits verbrauchen.', + 'memorySources.allIn.confirm': 'Ja', + 'memorySources.allIn.cancel': 'Nein', + 'memorySources.allIn.success': 'Alle Quellen ohne Limits aktiviert. Synchronisierung gestartet.', + 'memorySources.allIn.failed': 'Konnte „Alles aktivieren" nicht anwenden. Bitte erneut versuchen.', + 'memorySources.settings.button': 'Einstellungen', + 'memorySources.settings.title': 'Synchronisierungseinstellungen', + 'memorySources.settings.maxPrs': 'Maximale Pull-Requests', + 'memorySources.settings.maxIssues': 'Maximale Issues', + 'memorySources.settings.maxCommits': 'Maximale Commits', + 'memorySources.settings.maxItems': 'Maximale Elemente', + 'memorySources.settings.sinceDays': 'Rückblick (Tage)', + 'memorySources.settings.syncDepthDays': 'Synchronisierungstiefe (Tage)', + 'memorySources.settings.maxTokens': 'Maximale Tokens pro Synchronisierung', + 'memorySources.settings.maxCost': 'Maximale Kosten pro Synchronisierung (USD)', + 'memorySources.settings.unlimited': 'Unbegrenzt', + 'memorySources.settings.unlimitedTooltip': + 'Du hast dich entschieden, das Maximum für {toolkit} zu synchronisieren. Du kannst die Limits hier ändern.', + 'memorySources.settings.maxed': 'Voll', + 'memorySources.settings.save': 'Speichern', + 'memorySources.settings.saving': 'Speichern…', + 'memorySources.settings.saved': 'Einstellungen gespeichert', + 'memorySources.settings.saveFailed': 'Einstellungen konnten nicht gespeichert werden', 'backend.aiBackend': 'KI-Backend', 'backend.cloud': 'Wolke', 'backend.recommended': 'Empfohlen', diff --git a/app/src/lib/i18n/en.ts b/app/src/lib/i18n/en.ts index 6a2c70ed9..95450167b 100644 --- a/app/src/lib/i18n/en.ts +++ b/app/src/lib/i18n/en.ts @@ -2351,6 +2351,32 @@ const en: TranslationMap = { 'memorySources.build.successTitle': 'Tree built', 'memorySources.build.failedTitle': 'Build failed', 'memorySources.build.sealsMessage': 'seal(s) completed', + 'memorySources.allIn.button': 'All In', + 'memorySources.allIn.title': 'Go All In?', + 'memorySources.allIn.message': + 'This enables every memory source and removes all sync limits. It builds the richest memory graph, but may use more credits.', + 'memorySources.allIn.confirm': 'Yes', + 'memorySources.allIn.cancel': 'No', + 'memorySources.allIn.success': 'All sources enabled with no limits. Syncing started.', + 'memorySources.allIn.failed': 'Could not apply All In. Please try again.', + 'memorySources.settings.button': 'Settings', + 'memorySources.settings.title': 'Sync settings', + 'memorySources.settings.maxPrs': 'Max pull requests', + 'memorySources.settings.maxIssues': 'Max issues', + 'memorySources.settings.maxCommits': 'Max commits', + 'memorySources.settings.maxItems': 'Max items', + 'memorySources.settings.sinceDays': 'Lookback (days)', + 'memorySources.settings.syncDepthDays': 'Sync depth (days)', + 'memorySources.settings.maxTokens': 'Max tokens per sync', + 'memorySources.settings.maxCost': 'Max cost per sync (USD)', + 'memorySources.settings.unlimited': 'Unlimited', + 'memorySources.settings.unlimitedTooltip': + "You've opted in to sync the maximum for {toolkit}. You can change the caps here.", + 'memorySources.settings.maxed': 'Maxed', + 'memorySources.settings.save': 'Save', + 'memorySources.settings.saving': 'Saving…', + 'memorySources.settings.saved': 'Settings saved', + 'memorySources.settings.saveFailed': 'Could not save settings', // Backend 'backend.aiBackend': 'AI Backend', diff --git a/app/src/lib/i18n/es.ts b/app/src/lib/i18n/es.ts index e246a3c0a..ea9f67e4c 100644 --- a/app/src/lib/i18n/es.ts +++ b/app/src/lib/i18n/es.ts @@ -2064,6 +2064,33 @@ const messages: TranslationMap = { 'memorySources.build.successTitle': 'Árbol construido', 'memorySources.build.failedTitle': 'Construcción fallida', 'memorySources.build.sealsMessage': 'sellado(s) completado(s)', + 'memorySources.allIn.button': 'Todo activo', + 'memorySources.allIn.title': '¿Activar todo?', + 'memorySources.allIn.message': + 'Esto activará todas las fuentes de memoria y eliminará todos los límites de sincronización. Construye el gráfico de memoria más rico, pero puede usar más créditos.', + 'memorySources.allIn.confirm': 'Sí', + 'memorySources.allIn.cancel': 'No', + 'memorySources.allIn.success': + 'Todas las fuentes activadas sin límites. Sincronización iniciada.', + 'memorySources.allIn.failed': 'No se pudo aplicar todo activo. Por favor, inténtelo de nuevo.', + 'memorySources.settings.button': 'Configuración', + 'memorySources.settings.title': 'Configuración de sincronización', + 'memorySources.settings.maxPrs': 'Máximo de pull requests', + 'memorySources.settings.maxIssues': 'Máximo de issues', + 'memorySources.settings.maxCommits': 'Máximo de commits', + 'memorySources.settings.maxItems': 'Máximo de elementos', + 'memorySources.settings.sinceDays': 'Período de búsqueda (días)', + 'memorySources.settings.syncDepthDays': 'Profundidad de sincronización (días)', + 'memorySources.settings.maxTokens': 'Máximo de tokens por sincronización', + 'memorySources.settings.maxCost': 'Costo máximo por sincronización (USD)', + 'memorySources.settings.unlimited': 'Sin límite', + 'memorySources.settings.unlimitedTooltip': + 'Has optado por sincronizar el máximo para {toolkit}. Puedes cambiar los límites aquí.', + 'memorySources.settings.maxed': 'Lleno', + 'memorySources.settings.save': 'Guardar', + 'memorySources.settings.saving': 'Guardando…', + 'memorySources.settings.saved': 'Configuración guardada', + 'memorySources.settings.saveFailed': 'No se pudo guardar la configuración', 'backend.aiBackend': 'Backend de IA', 'backend.cloud': 'Nube', 'backend.recommended': 'Recomendado', diff --git a/app/src/lib/i18n/fr.ts b/app/src/lib/i18n/fr.ts index ee9136352..1e9862027 100644 --- a/app/src/lib/i18n/fr.ts +++ b/app/src/lib/i18n/fr.ts @@ -2070,6 +2070,32 @@ const messages: TranslationMap = { 'memorySources.build.successTitle': 'Arbre construit', 'memorySources.build.failedTitle': 'Échec de construction', 'memorySources.build.sealsMessage': 'scellement(s) terminé(s)', + 'memorySources.allIn.button': 'Tout activer', + 'memorySources.allIn.title': 'Tout activer ?', + 'memorySources.allIn.message': + 'Cela activera toutes les sources de mémoire et supprimera toutes les limites de synchronisation. Cela construit le graphe de mémoire le plus riche, mais peut utiliser davantage de crédits.', + 'memorySources.allIn.confirm': 'Oui', + 'memorySources.allIn.cancel': 'Non', + 'memorySources.allIn.success': 'Toutes les sources activées sans limite. Synchronisation lancée.', + 'memorySources.allIn.failed': "Impossible d'appliquer tout activer. Veuillez réessayer.", + 'memorySources.settings.button': 'Paramètres', + 'memorySources.settings.title': 'Paramètres de synchronisation', + 'memorySources.settings.maxPrs': 'Nombre maximal de pull requests', + 'memorySources.settings.maxIssues': "Nombre maximal d'issues", + 'memorySources.settings.maxCommits': 'Nombre maximal de commits', + 'memorySources.settings.maxItems': "Nombre maximal d'éléments", + 'memorySources.settings.sinceDays': 'Période de recherche (jours)', + 'memorySources.settings.syncDepthDays': 'Profondeur de synchronisation (jours)', + 'memorySources.settings.maxTokens': 'Tokens maximum par synchronisation', + 'memorySources.settings.maxCost': 'Coût maximum par synchronisation (USD)', + 'memorySources.settings.unlimited': 'Illimité', + 'memorySources.settings.unlimitedTooltip': + 'Vous avez choisi de synchroniser le maximum pour {toolkit}. Vous pouvez modifier les limites ici.', + 'memorySources.settings.maxed': 'Plein', + 'memorySources.settings.save': 'Enregistrer', + 'memorySources.settings.saving': 'Enregistrement…', + 'memorySources.settings.saved': 'Paramètres enregistrés', + 'memorySources.settings.saveFailed': "Impossible d'enregistrer les paramètres", 'backend.aiBackend': 'Backend IA', 'backend.cloud': 'Cloud', 'backend.recommended': 'Recommandé', diff --git a/app/src/lib/i18n/hi.ts b/app/src/lib/i18n/hi.ts index c6d10a476..30c063459 100644 --- a/app/src/lib/i18n/hi.ts +++ b/app/src/lib/i18n/hi.ts @@ -2022,6 +2022,32 @@ const messages: TranslationMap = { 'memorySources.build.successTitle': 'ट्री बन गई', 'memorySources.build.failedTitle': 'निर्माण विफल', 'memorySources.build.sealsMessage': 'सील पूर्ण हुई', + 'memorySources.allIn.button': 'सब चालू', + 'memorySources.allIn.title': 'सब कुछ सक्रिय करें?', + 'memorySources.allIn.message': + 'यह सभी मेमोरी स्रोत सक्रिय करेगा और सभी सिंक सीमाएं हटा देगा। यह सबसे समृद्ध मेमोरी ग्राफ बनाता है, लेकिन अधिक क्रेडिट उपयोग कर सकता है।', + 'memorySources.allIn.confirm': 'हाँ', + 'memorySources.allIn.cancel': 'नहीं', + 'memorySources.allIn.success': 'सभी स्रोत बिना सीमा के सक्रिय हुए। सिंक शुरू हो गई।', + 'memorySources.allIn.failed': 'सब चालू नहीं किया जा सका। कृपया पुनः प्रयास करें।', + 'memorySources.settings.button': 'सेटिंग', + 'memorySources.settings.title': 'सिंक सेटिंग', + 'memorySources.settings.maxPrs': 'अधिकतम पुल रिक्वेस्ट', + 'memorySources.settings.maxIssues': 'अधिकतम इश्यू', + 'memorySources.settings.maxCommits': 'अधिकतम कमिट', + 'memorySources.settings.maxItems': 'अधिकतम आइटम', + 'memorySources.settings.sinceDays': 'पिछले दिन (दिनों में)', + 'memorySources.settings.syncDepthDays': 'सिंक गहराई (दिनों में)', + 'memorySources.settings.maxTokens': 'प्रति सिंक अधिकतम टोकन', + 'memorySources.settings.maxCost': 'प्रति सिंक अधिकतम लागत (USD)', + 'memorySources.settings.unlimited': 'असीमित', + 'memorySources.settings.unlimitedTooltip': + 'आपने {toolkit} के लिए अधिकतम सिंक करना चुना है। आप यहाँ सीमाएँ बदल सकते हैं।', + 'memorySources.settings.maxed': 'पूर्ण', + 'memorySources.settings.save': 'सहेजें', + 'memorySources.settings.saving': 'सहेजा जा रहा है…', + 'memorySources.settings.saved': 'सेटिंग सहेजी गई', + 'memorySources.settings.saveFailed': 'सेटिंग सहेजी नहीं जा सकी', 'backend.aiBackend': 'AI बैकएंड', 'backend.cloud': 'क्लाउड', 'backend.recommended': 'सुझावित', diff --git a/app/src/lib/i18n/id.ts b/app/src/lib/i18n/id.ts index ea498c6d5..dddfe53bf 100644 --- a/app/src/lib/i18n/id.ts +++ b/app/src/lib/i18n/id.ts @@ -2025,6 +2025,32 @@ const messages: TranslationMap = { 'memorySources.build.successTitle': 'Pohon dibangun', 'memorySources.build.failedTitle': 'Pembangunan gagal', 'memorySources.build.sealsMessage': 'penyegelan selesai', + 'memorySources.allIn.button': 'Aktifkan Semua', + 'memorySources.allIn.title': 'Aktifkan Semua?', + 'memorySources.allIn.message': + 'Ini akan mengaktifkan semua sumber memori dan menghapus semua batas sinkronisasi. Membangun grafik memori terkaya, tetapi mungkin menggunakan lebih banyak kredit.', + 'memorySources.allIn.confirm': 'Ya', + 'memorySources.allIn.cancel': 'Tidak', + 'memorySources.allIn.success': 'Semua sumber diaktifkan tanpa batas. Sinkronisasi dimulai.', + 'memorySources.allIn.failed': 'Gagal mengaktifkan semua. Silakan coba lagi.', + 'memorySources.settings.button': 'Pengaturan', + 'memorySources.settings.title': 'Pengaturan sinkronisasi', + 'memorySources.settings.maxPrs': 'Maksimal pull request', + 'memorySources.settings.maxIssues': 'Maksimal isu', + 'memorySources.settings.maxCommits': 'Maksimal commit', + 'memorySources.settings.maxItems': 'Maksimal item', + 'memorySources.settings.sinceDays': 'Periode ke belakang (hari)', + 'memorySources.settings.syncDepthDays': 'Kedalaman sinkronisasi (hari)', + 'memorySources.settings.maxTokens': 'Maksimal token per sinkronisasi', + 'memorySources.settings.maxCost': 'Biaya maksimal per sinkronisasi (USD)', + 'memorySources.settings.unlimited': 'Tanpa batas', + 'memorySources.settings.unlimitedTooltip': + 'Anda memilih menyinkronkan maksimum untuk {toolkit}. Anda dapat mengubah batas di sini.', + 'memorySources.settings.maxed': 'Penuh', + 'memorySources.settings.save': 'Simpan', + 'memorySources.settings.saving': 'Menyimpan…', + 'memorySources.settings.saved': 'Pengaturan tersimpan', + 'memorySources.settings.saveFailed': 'Gagal menyimpan pengaturan', 'backend.aiBackend': 'Backend AI', 'backend.cloud': 'Awan', 'backend.recommended': 'Direkomendasikan', diff --git a/app/src/lib/i18n/it.ts b/app/src/lib/i18n/it.ts index 6a3f410a1..b252ea1a1 100644 --- a/app/src/lib/i18n/it.ts +++ b/app/src/lib/i18n/it.ts @@ -2054,6 +2054,33 @@ const messages: TranslationMap = { 'memorySources.build.successTitle': 'Albero costruito', 'memorySources.build.failedTitle': 'Costruzione fallita', 'memorySources.build.sealsMessage': 'sigillatura completata', + 'memorySources.allIn.button': 'Tutto attivo', + 'memorySources.allIn.title': 'Attivare tutto?', + 'memorySources.allIn.message': + 'Questo attiverà tutte le sorgenti di memoria e rimuoverà tutti i limiti di sincronizzazione. Costruisce il grafico di memoria più ricco, ma potrebbe utilizzare più crediti.', + 'memorySources.allIn.confirm': 'Sì', + 'memorySources.allIn.cancel': 'No', + 'memorySources.allIn.success': + 'Tutte le sorgenti attivate senza limiti. Sincronizzazione avviata.', + 'memorySources.allIn.failed': 'Impossibile attivare tutto. Riprova.', + 'memorySources.settings.button': 'Impostazioni', + 'memorySources.settings.title': 'Impostazioni di sincronizzazione', + 'memorySources.settings.maxPrs': 'Numero massimo di pull request', + 'memorySources.settings.maxIssues': 'Numero massimo di issue', + 'memorySources.settings.maxCommits': 'Numero massimo di commit', + 'memorySources.settings.maxItems': 'Numero massimo di elementi', + 'memorySources.settings.sinceDays': 'Periodo di ricerca (giorni)', + 'memorySources.settings.syncDepthDays': 'Profondità di sincronizzazione (giorni)', + 'memorySources.settings.maxTokens': 'Token massimi per sincronizzazione', + 'memorySources.settings.maxCost': 'Costo massimo per sincronizzazione (USD)', + 'memorySources.settings.unlimited': 'Illimitato', + 'memorySources.settings.unlimitedTooltip': + 'Hai scelto di sincronizzare il massimo per {toolkit}. Puoi modificare i limiti qui.', + 'memorySources.settings.maxed': 'Pieno', + 'memorySources.settings.save': 'Salva', + 'memorySources.settings.saving': 'Salvataggio…', + 'memorySources.settings.saved': 'Impostazioni salvate', + 'memorySources.settings.saveFailed': 'Impossibile salvare le impostazioni', 'backend.aiBackend': 'Backend AI', 'backend.cloud': 'Cloud', 'backend.recommended': 'Consigliato', diff --git a/app/src/lib/i18n/ko.ts b/app/src/lib/i18n/ko.ts index da2220a5f..4cb35fb44 100644 --- a/app/src/lib/i18n/ko.ts +++ b/app/src/lib/i18n/ko.ts @@ -1999,6 +1999,32 @@ const messages: TranslationMap = { 'memorySources.build.successTitle': '트리 빌드 완료', 'memorySources.build.failedTitle': '빌드 실패', 'memorySources.build.sealsMessage': '실링 완료', + 'memorySources.allIn.button': '전체 활성화', + 'memorySources.allIn.title': '전체 활성화할까요?', + 'memorySources.allIn.message': + '모든 메모리 소스를 활성화하고 동기화 제한을 제거합니다. 가장 풍부한 메모리 그래프를 구축하지만 크레딧을 더 사용할 수 있습니다.', + 'memorySources.allIn.confirm': '예', + 'memorySources.allIn.cancel': '아니오', + 'memorySources.allIn.success': '제한 없이 모든 소스가 활성화되었습니다. 동기화가 시작되었습니다.', + 'memorySources.allIn.failed': '전체 활성화를 적용하지 못했습니다. 다시 시도해 주세요.', + 'memorySources.settings.button': '설정', + 'memorySources.settings.title': '동기화 설정', + 'memorySources.settings.maxPrs': '최대 풀 리퀘스트 수', + 'memorySources.settings.maxIssues': '최대 이슈 수', + 'memorySources.settings.maxCommits': '최대 커밋 수', + 'memorySources.settings.maxItems': '최대 항목 수', + 'memorySources.settings.sinceDays': '조회 기간 (일)', + 'memorySources.settings.syncDepthDays': '동기화 깊이 (일)', + 'memorySources.settings.maxTokens': '동기화당 최대 토큰', + 'memorySources.settings.maxCost': '동기화당 최대 비용 (USD)', + 'memorySources.settings.unlimited': '무제한', + 'memorySources.settings.unlimitedTooltip': + '{toolkit}에 대해 최대로 동기화하도록 선택했습니다. 여기에서 한도를 변경할 수 있습니다.', + 'memorySources.settings.maxed': '최대', + 'memorySources.settings.save': '저장', + 'memorySources.settings.saving': '저장 중…', + 'memorySources.settings.saved': '설정이 저장되었습니다', + 'memorySources.settings.saveFailed': '설정을 저장하지 못했습니다', 'backend.aiBackend': 'AI 백엔드', 'backend.cloud': '클라우드', 'backend.recommended': '추천', diff --git a/app/src/lib/i18n/pl.ts b/app/src/lib/i18n/pl.ts index 08ba9c1c9..31056f76c 100644 --- a/app/src/lib/i18n/pl.ts +++ b/app/src/lib/i18n/pl.ts @@ -2043,6 +2043,33 @@ const messages: TranslationMap = { 'memorySources.build.successTitle': 'Drzewo zbudowane', 'memorySources.build.failedTitle': 'Budowanie nie powiodło się', 'memorySources.build.sealsMessage': 'pieczętowanie zakończone', + 'memorySources.allIn.button': 'Włącz wszystko', + 'memorySources.allIn.title': 'Włączyć wszystko?', + 'memorySources.allIn.message': + 'Spowoduje to włączenie wszystkich źródeł pamięci i usunięcie wszystkich limitów synchronizacji. Buduje najbogatszy wykres pamięci, ale może zużywać więcej kredytów.', + 'memorySources.allIn.confirm': 'Tak', + 'memorySources.allIn.cancel': 'Nie', + 'memorySources.allIn.success': + 'Wszystkie źródła włączone bez limitów. Synchronizacja uruchomiona.', + 'memorySources.allIn.failed': 'Nie udało się włączyć wszystkiego. Spróbuj ponownie.', + 'memorySources.settings.button': 'Ustawienia', + 'memorySources.settings.title': 'Ustawienia synchronizacji', + 'memorySources.settings.maxPrs': 'Maksymalna liczba pull requestów', + 'memorySources.settings.maxIssues': 'Maksymalna liczba zgłoszeń', + 'memorySources.settings.maxCommits': 'Maksymalna liczba commitów', + 'memorySources.settings.maxItems': 'Maksymalna liczba elementów', + 'memorySources.settings.sinceDays': 'Okres wsteczny (dni)', + 'memorySources.settings.syncDepthDays': 'Głębokość synchronizacji (dni)', + 'memorySources.settings.maxTokens': 'Maksymalna liczba tokenów na synchronizację', + 'memorySources.settings.maxCost': 'Maksymalny koszt na synchronizację (USD)', + 'memorySources.settings.unlimited': 'Bez limitu', + 'memorySources.settings.unlimitedTooltip': + 'Wybrano synchronizację maksimum dla {toolkit}. Limity możesz zmienić tutaj.', + 'memorySources.settings.maxed': 'Pełny', + 'memorySources.settings.save': 'Zapisz', + 'memorySources.settings.saving': 'Zapisywanie…', + 'memorySources.settings.saved': 'Ustawienia zapisane', + 'memorySources.settings.saveFailed': 'Nie udało się zapisać ustawień', 'backend.aiBackend': 'Backend AI', 'backend.cloud': 'Chmura', 'backend.recommended': 'Zalecane', diff --git a/app/src/lib/i18n/pt.ts b/app/src/lib/i18n/pt.ts index bcd358b6d..de690be01 100644 --- a/app/src/lib/i18n/pt.ts +++ b/app/src/lib/i18n/pt.ts @@ -2062,6 +2062,32 @@ const messages: TranslationMap = { 'memorySources.build.successTitle': 'Árvore construída', 'memorySources.build.failedTitle': 'Falha na construção', 'memorySources.build.sealsMessage': 'selagem concluída', + 'memorySources.allIn.button': 'Ativar tudo', + 'memorySources.allIn.title': 'Ativar tudo?', + 'memorySources.allIn.message': + 'Isso ativará todas as fontes de memória e removerá todos os limites de sincronização. Cria o grafo de memória mais rico, mas pode usar mais créditos.', + 'memorySources.allIn.confirm': 'Sim', + 'memorySources.allIn.cancel': 'Não', + 'memorySources.allIn.success': 'Todas as fontes ativadas sem limites. Sincronização iniciada.', + 'memorySources.allIn.failed': 'Não foi possível ativar tudo. Por favor, tente novamente.', + 'memorySources.settings.button': 'Configurações', + 'memorySources.settings.title': 'Configurações de sincronização', + 'memorySources.settings.maxPrs': 'Máximo de pull requests', + 'memorySources.settings.maxIssues': 'Máximo de issues', + 'memorySources.settings.maxCommits': 'Máximo de commits', + 'memorySources.settings.maxItems': 'Máximo de itens', + 'memorySources.settings.sinceDays': 'Período de busca (dias)', + 'memorySources.settings.syncDepthDays': 'Profundidade de sincronização (dias)', + 'memorySources.settings.maxTokens': 'Máximo de tokens por sincronização', + 'memorySources.settings.maxCost': 'Custo máximo por sincronização (USD)', + 'memorySources.settings.unlimited': 'Ilimitado', + 'memorySources.settings.unlimitedTooltip': + 'Você optou por sincronizar o máximo para {toolkit}. Você pode alterar os limites aqui.', + 'memorySources.settings.maxed': 'Cheio', + 'memorySources.settings.save': 'Salvar', + 'memorySources.settings.saving': 'Salvando…', + 'memorySources.settings.saved': 'Configurações salvas', + 'memorySources.settings.saveFailed': 'Não foi possível salvar as configurações', 'backend.aiBackend': 'Backend de IA', 'backend.cloud': 'Nuvem', 'backend.recommended': 'Recomendado', diff --git a/app/src/lib/i18n/ru.ts b/app/src/lib/i18n/ru.ts index a96152892..dc76e3702 100644 --- a/app/src/lib/i18n/ru.ts +++ b/app/src/lib/i18n/ru.ts @@ -2035,6 +2035,34 @@ const messages: TranslationMap = { 'memorySources.build.successTitle': 'Дерево построено', 'memorySources.build.failedTitle': 'Ошибка построения', 'memorySources.build.sealsMessage': 'запечатывание завершено', + 'memorySources.allIn.button': 'Включить всё', + 'memorySources.allIn.title': 'Включить всё?', + 'memorySources.allIn.message': + 'Это активирует все источники памяти и снимет все ограничения синхронизации. Создаёт наиболее богатый граф памяти, но может использовать больше кредитов.', + 'memorySources.allIn.confirm': 'Да', + 'memorySources.allIn.cancel': 'Нет', + 'memorySources.allIn.success': + 'Все источники активированы без ограничений. Синхронизация запущена.', + 'memorySources.allIn.failed': + 'Не удалось применить «Включить всё». Пожалуйста, попробуйте снова.', + 'memorySources.settings.button': 'Настройки', + 'memorySources.settings.title': 'Настройки синхронизации', + 'memorySources.settings.maxPrs': 'Максимум pull request', + 'memorySources.settings.maxIssues': 'Максимум задач', + 'memorySources.settings.maxCommits': 'Максимум коммитов', + 'memorySources.settings.maxItems': 'Максимум элементов', + 'memorySources.settings.sinceDays': 'Период (дней)', + 'memorySources.settings.syncDepthDays': 'Глубина синхронизации (дней)', + 'memorySources.settings.maxTokens': 'Максимум токенов за синхронизацию', + 'memorySources.settings.maxCost': 'Максимальная стоимость за синхронизацию (USD)', + 'memorySources.settings.unlimited': 'Без лимита', + 'memorySources.settings.unlimitedTooltip': + 'Вы выбрали синхронизацию максимума для {toolkit}. Лимиты можно изменить здесь.', + 'memorySources.settings.maxed': 'Заполнено', + 'memorySources.settings.save': 'Сохранить', + 'memorySources.settings.saving': 'Сохранение…', + 'memorySources.settings.saved': 'Настройки сохранены', + 'memorySources.settings.saveFailed': 'Не удалось сохранить настройки', 'backend.aiBackend': 'AI-бэкенд', 'backend.cloud': 'Облако', 'backend.recommended': 'Рекомендуется', diff --git a/app/src/lib/i18n/zh-CN.ts b/app/src/lib/i18n/zh-CN.ts index 18f7fa40f..15157d4d3 100644 --- a/app/src/lib/i18n/zh-CN.ts +++ b/app/src/lib/i18n/zh-CN.ts @@ -1921,6 +1921,32 @@ const messages: TranslationMap = { 'memorySources.build.successTitle': '树构建完成', 'memorySources.build.failedTitle': '构建失败', 'memorySources.build.sealsMessage': '密封完成', + 'memorySources.allIn.button': '全部启用', + 'memorySources.allIn.title': '全部启用？', + 'memorySources.allIn.message': + '这将启用所有记忆来源并移除所有同步限制。可构建最丰富的记忆图谱，但可能会消耗更多积分。', + 'memorySources.allIn.confirm': '是', + 'memorySources.allIn.cancel': '否', + 'memorySources.allIn.success': '所有来源已无限制启用。同步已开始。', + 'memorySources.allIn.failed': '无法应用全部启用。请重试。', + 'memorySources.settings.button': '设置', + 'memorySources.settings.title': '同步设置', + 'memorySources.settings.maxPrs': '最大拉取请求数', + 'memorySources.settings.maxIssues': '最大问题数', + 'memorySources.settings.maxCommits': '最大提交数', + 'memorySources.settings.maxItems': '最大条目数', + 'memorySources.settings.sinceDays': '回溯天数', + 'memorySources.settings.syncDepthDays': '同步深度（天）', + 'memorySources.settings.maxTokens': '每次同步最大令牌数', + 'memorySources.settings.maxCost': '每次同步最大费用（USD）', + 'memorySources.settings.unlimited': '无限制', + 'memorySources.settings.unlimitedTooltip': + '您已选择为 {toolkit} 同步最大数量。您可以在此处更改上限。', + 'memorySources.settings.maxed': '已满', + 'memorySources.settings.save': '保存', + 'memorySources.settings.saving': '保存中…', + 'memorySources.settings.saved': '设置已保存', + 'memorySources.settings.saveFailed': '无法保存设置', 'backend.aiBackend': 'AI 后端', 'backend.cloud': '云端', 'backend.recommended': '推荐', diff --git a/app/src/services/memorySourcesService.test.ts b/app/src/services/memorySourcesService.test.ts index 34300a3bb..cf2a499a1 100644 --- a/app/src/services/memorySourcesService.test.ts +++ b/app/src/services/memorySourcesService.test.ts @@ -3,7 +3,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { callCoreRpc } from './coreRpcClient'; import { addMemorySource, + applyAllIn, listMemorySources, + type MemorySourceEntry, removeMemorySource, SOURCE_KIND_ICONS, SOURCE_KIND_LABEL_KEYS, @@ -96,4 +98,73 @@ describe('memorySourcesService', () => { expect(SOURCE_KIND_ICONS[kind]).toBeTruthy(); } }); + + // --------------------------------------------------------------------------- + // applyAllIn + // --------------------------------------------------------------------------- + + it('applyAllIn calls the correct RPC method', async () => { + mockedCall.mockResolvedValue({ + result: { + sources: [{ id: 'src_1', kind: 'github_repo', label: 'All Repos', enabled: true }], + sync_triggered: 1, + }, + logs: [], + } as never); + + const result = await applyAllIn(); + + expect(mockedCall).toHaveBeenCalledWith({ method: 'openhuman.memory_sources_apply_all_in' }); + expect(result.sync_triggered).toBe(1); + expect(result.sources).toHaveLength(1); + expect(result.sources[0].id).toBe('src_1'); + }); + + it('applyAllIn handles envelope-wrapped response', async () => { + mockedCall.mockResolvedValue({ result: { sources: [], sync_triggered: 0 }, logs: [] } as never); + + const result = await applyAllIn(); + expect(result.sources).toEqual([]); + expect(result.sync_triggered).toBe(0); + }); + + it('applyAllIn handles flat (un-wrapped) response', async () => { + mockedCall.mockResolvedValue({ + sources: [{ id: 'src_2', kind: 'folder', label: 'Docs', enabled: true }], + sync_triggered: 1, + } as never); + + const result = await applyAllIn(); + expect(result.sources).toHaveLength(1); + }); + + // --------------------------------------------------------------------------- + // MemorySourceEntry interface includes all limit fields + // --------------------------------------------------------------------------- + + it('MemorySourceEntry interface accepts all limit fields at compile-time', () => { + // This test is a type-level assertion: if the fields are missing from the + // interface the assignment below would fail TypeScript compilation. + const entry: MemorySourceEntry = { + id: 'src_1', + kind: 'github_repo', + label: 'Test', + enabled: true, + max_commits: 100, + max_issues: 200, + max_prs: 50, + max_items: 500, + since_days: 30, + sync_depth_days: 90, + max_tokens_per_sync: 100_000, + max_cost_per_sync_usd: 1.5, + }; + expect(entry.max_commits).toBe(100); + expect(entry.max_issues).toBe(200); + expect(entry.max_prs).toBe(50); + expect(entry.since_days).toBe(30); + expect(entry.sync_depth_days).toBe(90); + expect(entry.max_tokens_per_sync).toBe(100_000); + expect(entry.max_cost_per_sync_usd).toBe(1.5); + }); }); diff --git a/app/src/services/memorySourcesService.ts b/app/src/services/memorySourcesService.ts index 7fe9291d9..f954320fe 100644 --- a/app/src/services/memorySourcesService.ts +++ b/app/src/services/memorySourcesService.ts @@ -31,9 +31,16 @@ export interface MemorySourceEntry { branch?: string; paths?: string[]; query?: string; + selector?: string; + // Sync limit fields (all optional; omit = use backend default / unlimited) since_days?: number; max_items?: number; - selector?: string; + max_commits?: number; + max_issues?: number; + max_prs?: number; + sync_depth_days?: number; + max_tokens_per_sync?: number; + max_cost_per_sync_usd?: number; } export interface SourceItem { @@ -159,6 +166,25 @@ export async function syncMemorySource(sourceId: string): Promise { }); } +export interface ApplyAllInResult { + sources: MemorySourceEntry[]; + sync_triggered: number; +} + +/** + * Enables every memory source, clears all per-source sync caps, and + * triggers a background sync for each. Equivalent to the UI "All In" + * action. Maps to `openhuman.memory_sources_apply_all_in`. + */ +export async function applyAllIn(): Promise { + log('apply_all_in'); + const resp = await callCoreRpc({ + method: 'openhuman.memory_sources_apply_all_in', + }); + const data = unwrap(resp); + return { sources: data.sources ?? [], sync_triggered: data.sync_triggered ?? 0 }; +} + /// i18n keys for each source kind's user-visible label. Resolve via /// `t(SOURCE_KIND_LABEL_KEYS[kind])` in components — keeping the keys /// as a constant lets the dialog kind-picker render the same labels diff --git a/src/bin/slack_backfill.rs b/src/bin/slack_backfill.rs index 35b416613..d66d99a27 100644 --- a/src/bin/slack_backfill.rs +++ b/src/bin/slack_backfill.rs @@ -451,6 +451,8 @@ async fn main() -> Result<()> { toolkit: conn.toolkit.clone(), connection_id: Some(conn.id.clone()), usage: Default::default(), + max_items: None, + sync_depth_days: None, }; match run_backfill_via_search(&ctx, cli.days).await { Ok(outcome) => { @@ -549,6 +551,8 @@ async fn main() -> Result<()> { toolkit: conn.toolkit.clone(), connection_id: Some(conn.id.clone()), usage: Default::default(), + max_items: None, + sync_depth_days: None, }; match provider.sync(&ctx, SyncReason::Manual).await { Ok(outcome) => { diff --git a/src/openhuman/about_app/catalog_data.rs b/src/openhuman/about_app/catalog_data.rs index 9c116e9ba..3dac28ac8 100644 --- a/src/openhuman/about_app/catalog_data.rs +++ b/src/openhuman/about_app/catalog_data.rs @@ -355,6 +355,25 @@ pub(super) const CAPABILITIES: &[Capability] = &[ status: CapabilityStatus::Beta, privacy: GITHUB_REPO_SOURCE, }, + Capability { + id: "intelligence.memory_source_sync_controls", + name: "Memory Source Sync Defaults & Controls", + domain: "memory_sources", + category: CapabilityCategory::Intelligence, + description: "Connected memory sources are enabled by default with conservative, \ + per-kind sync caps so the first sync stays cheap (e.g. Gmail ~100 recent emails, \ + GitHub repo 10 PRs / 10 issues / 50 commits, RSS 20 items). Each source row exposes \ + an inline settings panel to adjust the limit fields that apply to its kind \ + (max_items, sync_depth_days, max_prs/issues/commits, since_days). \ + An \"All In\" action enables every source and removes the caps to build the richest \ + memory graph, then triggers a full sync. Already-connected sources are migrated to \ + the new defaults once.", + how_to: "Intelligence > Memory Sources — toggle a source, open its gear for per-source \ + limits, or use \"All In\". Programmatic: openhuman.memory_sources_update and \ + openhuman.memory_sources_apply_all_in (RPC).", + status: CapabilityStatus::Beta, + privacy: LOCAL_RAW, + }, Capability { id: "intelligence.embedding_provider_config", name: "Configure Embedding Provider", diff --git a/src/openhuman/about_app/catalog_tests.rs b/src/openhuman/about_app/catalog_tests.rs index 40e35b9be..d7c215d87 100644 --- a/src/openhuman/about_app/catalog_tests.rs +++ b/src/openhuman/about_app/catalog_tests.rs @@ -106,6 +106,7 @@ fn catalog_includes_additional_user_facing_surfaces() { "intelligence.embedding_provider_config", "intelligence.embedding_provider_test", "intelligence.github_repo_memory_source", + "intelligence.memory_source_sync_controls", "conversation.subagent_mascots", ] { assert!( diff --git a/src/openhuman/composio/ops.rs b/src/openhuman/composio/ops.rs index fd1209f00..05a04594b 100644 --- a/src/openhuman/composio/ops.rs +++ b/src/openhuman/composio/ops.rs @@ -1264,6 +1264,8 @@ pub async fn composio_get_user_profile( toolkit: toolkit.clone(), connection_id: Some(connection_id.to_string()), usage: Default::default(), + max_items: None, + sync_depth_days: None, }; let profile = provider.fetch_user_profile(&ctx).await.map_err(|e| { @@ -1336,6 +1338,8 @@ pub async fn composio_refresh_all_identities( toolkit: toolkit.clone(), connection_id: Some(connection_id.clone()), usage: Default::default(), + max_items: None, + sync_depth_days: None, }; match provider.fetch_user_profile(&ctx).await { @@ -1433,6 +1437,8 @@ pub async fn composio_sync( toolkit: toolkit.clone(), connection_id: Some(connection_id.to_string()), usage: Default::default(), + max_items: None, + sync_depth_days: None, }; let started_at_ms = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) diff --git a/src/openhuman/config/schema/types.rs b/src/openhuman/config/schema/types.rs index 52c65a7f7..a90ba80e8 100644 --- a/src/openhuman/config/schema/types.rs +++ b/src/openhuman/config/schema/types.rs @@ -419,6 +419,18 @@ pub struct Config { #[serde(default)] pub model_registry: Vec, + + /// Migration version guard for `apply_composio_source_caps_migration`. + /// + /// The migration runs whenever this is `< CURRENT_CAPS_MIGRATION_VERSION` + /// (see `memory_sources::reconcile`), then is bumped to that version. Using a + /// monotonic version (rather than a bool) lets an improved migration re-run + /// once for installs that already ran an earlier revision. Defaults to `0` + /// (`#[serde(default)]`); the retired `composio_source_caps_migrated` bool is + /// silently ignored (Config does not `deny_unknown_fields`), so prior installs + /// re-run the current migration exactly once. + #[serde(default)] + pub composio_source_caps_migration_version: u32, } /// Shared default so `#[serde(default)]` and `Config::default()` stay in sync. @@ -738,6 +750,7 @@ impl Default for Config { onboarding_completed: false, chat_onboarding_completed: false, model_registry: Vec::new(), + composio_source_caps_migration_version: 0, } } } diff --git a/src/openhuman/memory_sources/mod.rs b/src/openhuman/memory_sources/mod.rs index 565d0917c..096e3559d 100644 --- a/src/openhuman/memory_sources/mod.rs +++ b/src/openhuman/memory_sources/mod.rs @@ -25,10 +25,11 @@ pub mod sync; pub mod types; pub use registry::{ - add_source, get_source, list_enabled_by_kind, list_sources, - remove_composio_source_by_connection_id, remove_source, update_source, upsert_composio_source, - MemorySourcePatch, + add_source, apply_all_in, get_source, list_enabled_by_kind, list_sources, + memory_sync_defaults_for_toolkit, remove_composio_source_by_connection_id, remove_source, + update_source, upsert_composio_source, MemorySourcePatch, }; +pub use rpc::apply_kind_defaults; pub use schemas::{ all_controller_schemas as all_memory_sources_controller_schemas, all_registered_controllers as all_memory_sources_registered_controllers, diff --git a/src/openhuman/memory_sources/reconcile.rs b/src/openhuman/memory_sources/reconcile.rs index 26689774f..798d2fb8b 100644 --- a/src/openhuman/memory_sources/reconcile.rs +++ b/src/openhuman/memory_sources/reconcile.rs @@ -3,11 +3,20 @@ //! Called once at boot to ensure all active Composio sync targets have //! a corresponding `MemorySourceEntry` in config. This catches connections //! created before the memory_sources domain existed. +//! +//! Also owns the retroactive caps migration +//! (`apply_composio_source_caps_migration`) that gives any cap-less Composio +//! source — enabled or disabled — conservative per-toolkit caps. use crate::openhuman::config::rpc as config_rpc; use crate::openhuman::memory_sources::registry; +use crate::openhuman::memory_sources::types::{MemorySourceEntry, SourceKind}; use crate::openhuman::memory_sync::composio; +/// Current version of the caps migration. Bump when the migration logic changes +/// so installs that ran an earlier revision re-run it exactly once. +const CURRENT_CAPS_MIGRATION_VERSION: u32 = 1; + pub async fn ensure_composio_sources() { tracing::debug!("[memory_sources:reconcile] starting composio reconciliation"); @@ -67,6 +76,100 @@ pub async fn ensure_composio_sources() { "[memory_sources:reconcile] composio reconciliation complete" ); } + + // Run the one-time caps migration after the reconcile loop so any + // sources upserted just above are also considered. + if let Err(e) = apply_composio_source_caps_migration().await { + tracing::warn!( + error = %e, + "[memory_sources:reconcile] caps migration failed (non-fatal, will retry next time)" + ); + } +} + +/// Apply conservative default caps in-place to every cap-less source. +/// +/// For a Composio source with no `max_items`/`sync_depth_days`, writes the +/// per-toolkit defaults and enables it (a no-op when already enabled) — an +/// already-enabled, cap-less source would otherwise sync at the provider's large +/// internal ceiling instead of the cheap default. For other kinds, fills any unset +/// kind-specific caps via `apply_kind_defaults`. User-customised caps (non-None) +/// are never overwritten. Returns the number of Composio entries that received +/// defaults. Pure (no I/O) so it can be unit-tested directly. +fn apply_caps_defaults_to_entries(sources: &mut [MemorySourceEntry]) -> u32 { + let mut applied = 0u32; + for source in sources.iter_mut() { + match source.kind { + SourceKind::Composio => { + // Apply to enabled AND disabled cap-less sources; skip entries the + // user has already customised (any non-None cap). + if source.max_items.is_none() && source.sync_depth_days.is_none() { + let toolkit = source.toolkit.as_deref().unwrap_or(""); + let (max_items, sync_depth_days) = + registry::memory_sync_defaults_for_toolkit(toolkit); + tracing::debug!( + id = %source.id, + toolkit = %toolkit, + was_enabled = source.enabled, + max_items = ?max_items, + sync_depth_days = ?sync_depth_days, + "[memory_sources:reconcile] caps migration: applying conservative defaults" + ); + source.enabled = true; + source.max_items = max_items; + source.sync_depth_days = sync_depth_days; + applied += 1; + } + } + // Apply non-composio kind defaults for entries with all-None caps. + _ => { + // Use the rpc::apply_kind_defaults helper so the same + // conservative values are applied consistently. + crate::openhuman::memory_sources::rpc::apply_kind_defaults(source); + } + } + } + applied +} + +/// Retroactive migration: give any cap-less Composio source — enabled or +/// disabled — conservative per-toolkit caps so its first sync stays cheap. +/// +/// Version-gated by `Config.composio_source_caps_migration_version`: runs once per +/// `CURRENT_CAPS_MIGRATION_VERSION` bump (installs that ran an earlier revision +/// re-run it exactly once). Entries the user has already customised (non-None caps) +/// are left untouched. +pub async fn apply_composio_source_caps_migration() -> Result<(), String> { + let mut config = config_rpc::load_config_with_timeout().await?; + + if config.composio_source_caps_migration_version >= CURRENT_CAPS_MIGRATION_VERSION { + tracing::debug!( + version = config.composio_source_caps_migration_version, + "[memory_sources:reconcile] caps migration already at current version; skipping" + ); + return Ok(()); + } + + tracing::info!( + from_version = config.composio_source_caps_migration_version, + to_version = CURRENT_CAPS_MIGRATION_VERSION, + "[memory_sources:reconcile] applying composio source caps migration" + ); + + let migrated_count = apply_caps_defaults_to_entries(&mut config.memory_sources); + + config.composio_source_caps_migration_version = CURRENT_CAPS_MIGRATION_VERSION; + config + .save() + .await + .map_err(|e| format!("caps migration: failed to save config: {e:#}"))?; + + tracing::info!( + migrated = migrated_count, + "[memory_sources:reconcile] caps migration complete" + ); + + Ok(()) } fn title_case(s: &str) -> String { @@ -92,6 +195,109 @@ fn short_id(id: &str) -> &str { #[cfg(test)] mod tests { use super::*; + use crate::openhuman::memory_sources::types::{MemorySourceEntry, SourceKind}; + + fn make_composio_entry( + id: &str, + toolkit: &str, + enabled: bool, + max_items: Option, + sync_depth_days: Option, + ) -> MemorySourceEntry { + MemorySourceEntry { + id: id.to_string(), + kind: SourceKind::Composio, + label: toolkit.to_string(), + enabled, + toolkit: Some(toolkit.to_string()), + connection_id: Some(format!("conn_{id}")), + path: None, + glob: None, + url: None, + branch: None, + paths: Vec::new(), + max_commits: None, + max_issues: None, + max_prs: None, + query: None, + since_days: None, + max_items, + selector: None, + max_tokens_per_sync: None, + max_cost_per_sync_usd: None, + sync_depth_days, + } + } + + /// Exercises the real migration transform (`apply_caps_defaults_to_entries`) + /// so the tests cannot drift from the production predicate. + fn run_migration_on_entries(sources: &mut Vec) -> u32 { + apply_caps_defaults_to_entries(sources) + } + + #[test] + fn migration_flips_disabled_capless_entry_to_enabled_with_caps() { + let mut sources = vec![make_composio_entry("s1", "gmail", false, None, None)]; + let count = run_migration_on_entries(&mut sources); + assert_eq!(count, 1); + assert!(sources[0].enabled); + assert_eq!(sources[0].max_items, Some(100)); + assert_eq!(sources[0].sync_depth_days, Some(30)); + } + + #[test] + fn migration_applies_defaults_to_enabled_capless_entry() { + // An already-enabled but cap-less source must also receive defaults — + // otherwise its first sync runs at the provider's large internal ceiling. + let mut sources = vec![make_composio_entry("s2", "slack", true, None, None)]; + let count = run_migration_on_entries(&mut sources); + assert_eq!(count, 1); + assert!(sources[0].enabled); + assert_eq!(sources[0].max_items, Some(50)); + assert_eq!(sources[0].sync_depth_days, Some(14)); + } + + #[test] + fn migration_leaves_user_customised_caps_untouched() { + // User set max_items explicitly → migration should not override. + let mut sources = vec![make_composio_entry("s3", "notion", false, Some(5), None)]; + let count = run_migration_on_entries(&mut sources); + assert_eq!(count, 0, "entry with user-set caps must not be migrated"); + assert!(!sources[0].enabled, "enabled must not be flipped"); + assert_eq!(sources[0].max_items, Some(5), "user cap must be preserved"); + } + + #[test] + fn migration_is_noop_on_empty_list() { + let mut sources: Vec = vec![]; + let count = run_migration_on_entries(&mut sources); + assert_eq!(count, 0); + } + + #[test] + fn migration_applies_correct_defaults_per_toolkit() { + let toolkits = [ + ("gmail", Some(100u32), Some(30u32)), + ("slack", Some(50), Some(14)), + ("notion", Some(30), Some(30)), + ("linear", Some(50), Some(30)), + ("clickup", Some(50), Some(30)), + ("github", Some(50), Some(30)), + ("unknown", Some(30), Some(14)), + ]; + for (toolkit, exp_items, exp_days) in &toolkits { + let mut sources = vec![make_composio_entry("sid", toolkit, false, None, None)]; + run_migration_on_entries(&mut sources); + assert_eq!( + sources[0].max_items, *exp_items, + "max_items mismatch for toolkit={toolkit}" + ); + assert_eq!( + sources[0].sync_depth_days, *exp_days, + "sync_depth_days mismatch for toolkit={toolkit}" + ); + } + } #[test] fn short_id_truncates_ascii() { diff --git a/src/openhuman/memory_sources/registry.rs b/src/openhuman/memory_sources/registry.rs index bbfdedf28..a362d7021 100644 --- a/src/openhuman/memory_sources/registry.rs +++ b/src/openhuman/memory_sources/registry.rs @@ -7,6 +7,27 @@ use crate::openhuman::config::rpc as config_rpc; use crate::openhuman::memory_sources::types::{MemorySourceEntry, SourceKind}; +/// Conservative default sync caps for a Composio toolkit, keyed by toolkit slug. +/// +/// Single source of truth for the cheap out-of-the-box sync volume. Applied to a +/// source entry when it is first registered (`upsert_composio_source`, insert-only) +/// and by the one-time caps migration (`reconcile::apply_composio_source_caps_migration`) +/// for cap-less entries. Never overwrites a user-customised cap. +/// +/// Returns `(max_items, sync_depth_days)`. +pub fn memory_sync_defaults_for_toolkit(toolkit: &str) -> (Option, Option) { + match toolkit { + "gmail" => (Some(100), Some(30)), + "slack" => (Some(50), Some(14)), + "notion" => (Some(30), Some(30)), + "linear" => (Some(50), Some(30)), + "clickup" => (Some(50), Some(30)), + "github" => (Some(50), Some(30)), + // Generic fallback for any toolkit not listed above. + _ => (Some(30), Some(14)), + } +} + pub async fn list_sources() -> Result, String> { let config = config_rpc::load_config_with_timeout().await?; Ok(config.memory_sources.clone()) @@ -38,7 +59,6 @@ pub async fn add_source(entry: MemorySourceEntry) -> Result, #[serde(default)] pub sync_depth_days: Option, + // ── GithubRepo-specific caps (previously missing from patch) ── + #[serde(default)] + pub max_commits: Option, + #[serde(default)] + pub max_issues: Option, + #[serde(default)] + pub max_prs: Option, +} + +/// Enable ALL configured memory sources and clear every per-source cap, +/// giving the user unrestricted access ("All In" mode). +/// +/// For each source in `config.memory_sources`: +/// - Sets `enabled = true`. +/// - Clears `max_items`, `since_days`, `sync_depth_days`, +/// `max_commits`, `max_issues`, `max_prs`, +/// `max_tokens_per_sync`, `max_cost_per_sync_usd` to `None`. +/// +/// Saves config once after all mutations and returns the updated entries. +pub async fn apply_all_in() -> Result, String> { + let mut config = config_rpc::load_config_with_timeout().await?; + + tracing::info!( + count = config.memory_sources.len(), + "[memory_sources] apply_all_in: enabling all sources and clearing caps" + ); + + for source in &mut config.memory_sources { + tracing::debug!( + id = %source.id, + kind = %source.kind.as_str(), + "[memory_sources] apply_all_in: enabling source and clearing caps" + ); + source.enabled = true; + source.max_items = None; + source.since_days = None; + source.sync_depth_days = None; + source.max_commits = None; + source.max_issues = None; + source.max_prs = None; + source.max_tokens_per_sync = None; + source.max_cost_per_sync_usd = None; + } + + let updated = config.memory_sources.clone(); + + config + .save() + .await + .map_err(|e| format!("apply_all_in: failed to save config: {e:#}"))?; + + tracing::info!( + count = updated.len(), + "[memory_sources] apply_all_in: complete — all sources enabled, all caps cleared" + ); + + Ok(updated) } #[cfg(test)] mod tests { use super::*; + #[test] + fn composio_defaults_for_known_toolkits() { + assert_eq!( + memory_sync_defaults_for_toolkit("gmail"), + (Some(100), Some(30)) + ); + assert_eq!( + memory_sync_defaults_for_toolkit("slack"), + (Some(50), Some(14)) + ); + assert_eq!( + memory_sync_defaults_for_toolkit("notion"), + (Some(30), Some(30)) + ); + assert_eq!( + memory_sync_defaults_for_toolkit("linear"), + (Some(50), Some(30)) + ); + assert_eq!( + memory_sync_defaults_for_toolkit("clickup"), + (Some(50), Some(30)) + ); + assert_eq!( + memory_sync_defaults_for_toolkit("github"), + (Some(50), Some(30)) + ); + } + + #[test] + fn composio_defaults_for_generic_fallback() { + assert_eq!( + memory_sync_defaults_for_toolkit("unknown_toolkit_xyz"), + (Some(30), Some(14)) + ); + assert_eq!(memory_sync_defaults_for_toolkit(""), (Some(30), Some(14))); + } + #[test] fn memory_source_patch_deserializes_partial() { let json = serde_json::json!({ "label": "New label", "enabled": false }); @@ -289,4 +420,29 @@ mod tests { assert_eq!(patch.enabled, Some(false)); assert!(patch.toolkit.is_none()); } + + #[test] + fn memory_source_patch_round_trips_github_limit_fields() { + let json = serde_json::json!({ + "max_commits": 100, + "max_issues": 50, + "max_prs": 25 + }); + let patch: MemorySourcePatch = serde_json::from_value(json).unwrap(); + assert_eq!(patch.max_commits, Some(100)); + assert_eq!(patch.max_issues, Some(50)); + assert_eq!(patch.max_prs, Some(25)); + // Unset fields must be None (serde(default)) + assert!(patch.label.is_none()); + assert!(patch.enabled.is_none()); + } + + #[test] + fn memory_source_patch_defaults_github_fields_to_none() { + let json = serde_json::json!({ "enabled": true }); + let patch: MemorySourcePatch = serde_json::from_value(json).unwrap(); + assert!(patch.max_commits.is_none()); + assert!(patch.max_issues.is_none()); + assert!(patch.max_prs.is_none()); + } } diff --git a/src/openhuman/memory_sources/rpc.rs b/src/openhuman/memory_sources/rpc.rs index 367b87de0..431f5eca8 100644 --- a/src/openhuman/memory_sources/rpc.rs +++ b/src/openhuman/memory_sources/rpc.rs @@ -105,7 +105,7 @@ pub async fn add_rpc(req: AddRequest) -> Result, String> "[memory_sources] add_rpc: entry" ); - let entry = MemorySourceEntry { + let mut entry = MemorySourceEntry { id: format!("src_{}", uuid::Uuid::new_v4().as_simple()), kind: req.kind, label: req.label, @@ -117,9 +117,6 @@ pub async fn add_rpc(req: AddRequest) -> Result, String> url: req.url, branch: req.branch, paths: req.paths, - // Per-type GitHub sync caps default to DEFAULT_GITHUB_ITEM_LIMIT - // (2000) in the reader when None. Overrides are applied via the - // update/patch path (`MemorySourcePatch`). max_commits: req.max_commits, max_issues: req.max_issues, max_prs: req.max_prs, @@ -132,10 +129,48 @@ pub async fn add_rpc(req: AddRequest) -> Result, String> sync_depth_days: req.sync_depth_days, }; + // Apply conservative per-kind defaults when the caller left caps unset. + apply_kind_defaults(&mut entry); + let source = registry::add_source(entry).await?; Ok(RpcOutcome::new(AddResponse { source }, vec![])) } +/// Apply conservative per-kind cap defaults to a new source entry. +/// +/// Only fills fields that are still `None` — never overwrites a +/// caller-supplied value. This mirrors the retroactive migration logic in +/// `reconcile::apply_composio_source_caps_migration` so the same defaults +/// are applied consistently at creation time and during migration. +pub fn apply_kind_defaults(entry: &mut MemorySourceEntry) { + match entry.kind { + SourceKind::GithubRepo => { + if entry.max_prs.is_none() { + entry.max_prs = Some(10); + } + if entry.max_issues.is_none() { + entry.max_issues = Some(10); + } + if entry.max_commits.is_none() { + entry.max_commits = Some(50); + } + } + SourceKind::RssFeed => { + if entry.max_items.is_none() { + entry.max_items = Some(20); + } + } + SourceKind::TwitterQuery => { + if entry.since_days.is_none() { + entry.since_days = Some(7); + } + } + // Folder / WebPage / Composio: no defaults to apply here. + // Composio defaults are set at upsert time in registry::upsert_composio_source. + _ => {} + } +} + // ── Update ── #[derive(Debug, serde::Deserialize)] @@ -394,3 +429,72 @@ pub async fn monthly_cost_summary_rpc() -> Result, + /// Number of sync tasks spawned (one per enabled source). + pub sync_triggered: u32, +} + +/// Enable ALL memory sources, clear all caps, and trigger a sync for +/// every source. +/// +/// Returns immediately with the updated source list and the number of +/// syncs queued. Individual syncs run in the background and publish +/// `MemorySyncStageChanged` events as they progress. +pub async fn apply_all_in_rpc() -> Result, String> { + tracing::info!("[memory_sources] apply_all_in_rpc: entry"); + + // Enable all sources and clear caps. + let sources = registry::apply_all_in().await?; + + // Trigger a background sync for every enabled source. + let config = config_rpc::load_config_with_timeout().await?; + let mut sync_triggered: u32 = 0; + + for source in &sources { + if !source.enabled { + continue; + } + tracing::debug!( + source_id = %source.id, + kind = %source.kind.as_str(), + "[memory_sources] apply_all_in_rpc: triggering sync" + ); + match crate::openhuman::memory_sources::sync::sync_source(source.clone(), config.clone()) + .await + { + Ok(()) => { + sync_triggered += 1; + } + Err(e) => { + // Non-fatal: log and continue — best-effort sync trigger. + tracing::warn!( + source_id = %source.id, + error = %e, + "[memory_sources] apply_all_in_rpc: sync trigger failed for source" + ); + } + } + } + + tracing::info!( + sources = sources.len(), + sync_triggered, + "[memory_sources] apply_all_in_rpc: complete" + ); + + Ok(RpcOutcome::new( + AllInResponse { + sources, + sync_triggered, + }, + vec![], + )) +} diff --git a/src/openhuman/memory_sources/schemas.rs b/src/openhuman/memory_sources/schemas.rs index 7184e2c45..4e5403275 100644 --- a/src/openhuman/memory_sources/schemas.rs +++ b/src/openhuman/memory_sources/schemas.rs @@ -55,6 +55,24 @@ fn kind_specific_fields() -> Vec { comment: "Path filters for github_repo sources.", required: false, }, + FieldSchema { + name: "max_commits", + ty: TypeSchema::Option(Box::new(TypeSchema::U64)), + comment: "Max commits per sync for github_repo sources.", + required: false, + }, + FieldSchema { + name: "max_issues", + ty: TypeSchema::Option(Box::new(TypeSchema::U64)), + comment: "Max issues per sync for github_repo sources.", + required: false, + }, + FieldSchema { + name: "max_prs", + ty: TypeSchema::Option(Box::new(TypeSchema::U64)), + comment: "Max pull requests per sync for github_repo sources.", + required: false, + }, FieldSchema { name: "query", ty: TypeSchema::Option(Box::new(TypeSchema::String)), @@ -70,7 +88,7 @@ fn kind_specific_fields() -> Vec { FieldSchema { name: "max_items", ty: TypeSchema::Option(Box::new(TypeSchema::U64)), - comment: "Maximum items for rss_feed sources.", + comment: "Maximum items for rss_feed or composio sources.", required: false, }, FieldSchema { @@ -114,6 +132,7 @@ pub fn all_controller_schemas() -> Vec { schemas("sync_audit_log"), schemas("estimate_sync_cost"), schemas("monthly_cost_summary"), + schemas("apply_all_in"), ] } @@ -167,6 +186,10 @@ pub fn all_registered_controllers() -> Vec { schema: schemas("monthly_cost_summary"), handler: handle_monthly_cost_summary, }, + RegisteredController { + schema: schemas("apply_all_in"), + handler: handle_apply_all_in, + }, ] } @@ -487,6 +510,29 @@ pub fn schemas(function: &str) -> ControllerSchema { }, ], }, + "apply_all_in" => ControllerSchema { + namespace: NAMESPACE, + function: "apply_all_in", + description: "Enable ALL memory sources, clear all per-source caps, \ + and trigger a background sync for every source. \ + Returns immediately with the updated source list and \ + the count of sync tasks queued.", + inputs: vec![], + outputs: vec![ + FieldSchema { + name: "sources", + ty: TypeSchema::Array(Box::new(TypeSchema::Ref("MemorySourceEntry"))), + comment: "All memory sources after the all-in transformation.", + required: true, + }, + FieldSchema { + name: "sync_triggered", + ty: TypeSchema::U64, + comment: "Number of sync tasks spawned.", + required: true, + }, + ], + }, other => panic!("unknown memory_sources schema function: {other}"), } } @@ -563,6 +609,10 @@ fn handle_monthly_cost_summary(_params: Map) -> ControllerFuture Box::pin(async move { to_json(rpc::monthly_cost_summary_rpc().await?) }) } +fn handle_apply_all_in(_params: Map) -> ControllerFuture { + Box::pin(async move { to_json(rpc::apply_all_in_rpc().await?) }) +} + fn parse_value(v: Value) -> Result { serde_json::from_value(v).map_err(|e| format!("invalid params: {e}")) } diff --git a/src/openhuman/memory_sync/composio/bus.rs b/src/openhuman/memory_sync/composio/bus.rs index ee1f84176..1b12b525a 100644 --- a/src/openhuman/memory_sync/composio/bus.rs +++ b/src/openhuman/memory_sync/composio/bus.rs @@ -470,7 +470,33 @@ impl EventHandler for ComposioConnectionCreatedSubscriber { return; } }; - let Some(ctx) = ProviderContext::from_config( + // Look up per-source caps from the memory_sources registry. + // Non-fatal: if the lookup fails we proceed without caps. + // + // upsert_composio_source runs AFTER this block (below), so for + // brand-new connections the entry may not exist yet. In that case + // fall back to the per-toolkit defaults so the first sync is still + // capped. list_enabled_by_kind would also drop disabled-but- + // configured entries, so we use list_sources() and filter ourselves. + let (src_max_items, src_sync_depth_days) = { + let registry_sources = crate::openhuman::memory_sources::list_sources() + .await + .unwrap_or_default(); + registry_sources + .iter() + .find(|s| { + s.kind == crate::openhuman::memory_sources::SourceKind::Composio + && s.connection_id.as_deref() == Some(connection_id.as_str()) + }) + .map(|s| (s.max_items, s.sync_depth_days)) + .unwrap_or_else(|| { + crate::openhuman::memory_sources::memory_sync_defaults_for_toolkit( + toolkit.as_str(), + ) + }) + }; + + let Some(mut ctx) = ProviderContext::from_config( Arc::new(config), toolkit.clone(), Some(connection_id.clone()), @@ -482,6 +508,17 @@ impl EventHandler for ComposioConnectionCreatedSubscriber { return; }; + ctx.max_items = src_max_items; + ctx.sync_depth_days = src_sync_depth_days; + + tracing::debug!( + toolkit = %toolkit, + connection_id = %connection_id, + max_items = ?src_max_items, + sync_depth_days = ?src_sync_depth_days, + "[composio:bus] caps from registry for connection_created" + ); + // `wait_for_connection_active` is a backend-only metadata // probe (`list_connections`). Resolve a backend // `ComposioClient` from the live config for it; direct-mode diff --git a/src/openhuman/memory_sync/composio/mod.rs b/src/openhuman/memory_sync/composio/mod.rs index 565fade10..6e20dcad8 100644 --- a/src/openhuman/memory_sync/composio/mod.rs +++ b/src/openhuman/memory_sync/composio/mod.rs @@ -151,11 +151,35 @@ pub async fn run_connection_sync( )) })?; + // Look up the source entry to obtain any user-configured caps. + // Non-fatal: if the registry read fails we proceed uncapped. + let (src_max_items, src_sync_depth_days) = { + let registry_sources = crate::openhuman::memory_sources::list_enabled_by_kind( + crate::openhuman::memory_sources::SourceKind::Composio, + ) + .await + .unwrap_or_default(); + registry_sources + .iter() + .find(|s| s.connection_id.as_deref() == Some(&target.connection_id)) + .map(|s| (s.max_items, s.sync_depth_days)) + .unwrap_or((None, None)) + }; + + tracing::debug!( + connection_id = %target.connection_id, + max_items = ?src_max_items, + sync_depth_days = ?src_sync_depth_days, + "[composio:sync] run_connection_sync: caps from registry" + ); + let ctx = ProviderContext { config: std::sync::Arc::new(config), toolkit: target.toolkit, connection_id: Some(target.connection_id), usage: Default::default(), + max_items: src_max_items, + sync_depth_days: src_sync_depth_days, }; let sync_result = provider.sync(&ctx, reason).await; diff --git a/src/openhuman/memory_sync/composio/periodic.rs b/src/openhuman/memory_sync/composio/periodic.rs index d0c69e5fb..557d031ac 100644 --- a/src/openhuman/memory_sync/composio/periodic.rs +++ b/src/openhuman/memory_sync/composio/periodic.rs @@ -313,6 +313,29 @@ pub(crate) async fn run_one_tick() -> Result<(), String> { continue; } + // Look up per-source caps from the memory_sources registry. + // Non-fatal: if the lookup fails we proceed without caps. + let (src_max_items, src_sync_depth_days) = { + let registry_sources = crate::openhuman::memory_sources::list_enabled_by_kind( + crate::openhuman::memory_sources::SourceKind::Composio, + ) + .await + .unwrap_or_default(); + registry_sources + .iter() + .find(|s| s.connection_id.as_deref() == Some(conn.id.as_str())) + .map(|s| (s.max_items, s.sync_depth_days)) + .unwrap_or((None, None)) + }; + + tracing::debug!( + toolkit = %toolkit, + connection_id = %conn.id, + max_items = ?src_max_items, + sync_depth_days = ?src_sync_depth_days, + "[composio:periodic] caps from registry" + ); + // Build a context tied to this specific connection and dispatch. // `ProviderContext` no longer caches a pre-baked // `ComposioClient` — provider methods resolve a fresh handle per @@ -323,6 +346,8 @@ pub(crate) async fn run_one_tick() -> Result<(), String> { toolkit: toolkit.clone(), connection_id: Some(conn.id.clone()), usage: Default::default(), + max_items: src_max_items, + sync_depth_days: src_sync_depth_days, }; tracing::debug!( diff --git a/src/openhuman/memory_sync/composio/providers/clickup/provider.rs b/src/openhuman/memory_sync/composio/providers/clickup/provider.rs index 0a8db0447..6688290e2 100644 --- a/src/openhuman/memory_sync/composio/providers/clickup/provider.rs +++ b/src/openhuman/memory_sync/composio/providers/clickup/provider.rs @@ -261,12 +261,42 @@ impl ComposioProvider for ClickUpProvider { _ => PAGE_SIZE, }; + // ctx.max_items: route through ItemCap — page ceiling, mid-page + // per-item break, and post-page hard stop all share one source of truth. + let mut cap = super::super::helpers::ItemCap::new(ctx.max_items); + let effective_max_pages = cap.max_pages(page_size, MAX_PAGES_PER_WORKSPACE); + if ctx.max_items.is_some() && effective_max_pages < MAX_PAGES_PER_WORKSPACE { + tracing::debug!( + connection_id = %connection_id, + max_items = ?ctx.max_items, + effective_max_pages, + "[composio:clickup] [memory_sync] applying max_items page cap" + ); + } + + // ctx.sync_depth_days: oldest allowed date_updated for client-side skip. + // ClickUp's `date_updated` field is a millisecond-epoch string, so the + // floor must also be epoch-millis (not RFC3339) for the lexicographic + // compare in `select_pending` to work correctly. + let oldest_allowed_time: Option = ctx.sync_depth_days.map(|days| { + let floor = chrono::Utc::now() - chrono::Duration::days(days as i64); + let s = floor.timestamp_millis().to_string(); + tracing::debug!( + connection_id = %connection_id, + sync_depth_days = days, + oldest_allowed_ms = %s, + "[composio:clickup] [memory_sync] applying sync_depth_days floor" + ); + s + }); + let mut total_fetched: usize = 0; let mut total_persisted: usize = 0; let mut newest_updated: Option = None; + let mut hit_cap_boundary = false; 'workspaces: for workspace_id in &workspaces { - for page_num in 0..MAX_PAGES_PER_WORKSPACE { + for page_num in 0..effective_max_pages { if state.budget_exhausted() { tracing::info!( workspace_id = %workspace_id, @@ -327,9 +357,27 @@ impl ComposioProvider for ClickUpProvider { } // ── Per-item dedup + bounded-concurrency ingest ───── - let (pending, hit_cursor_boundary) = + let (mut pending, mut hit_cursor_boundary) = select_pending(&tasks, &state, &mut newest_updated); + // ctx.sync_depth_days: drop items updated before the depth floor. `pending` is + // in descending timestamp order, so truncate at the first item below the floor + // and signal cursor-boundary so pagination stops. + if let Some(ref floor) = oldest_allowed_time { + if let Some(cut) = pending.iter().position(|p| { + p.updated + .as_deref() + .map(|t| t < floor.as_str()) + .unwrap_or(false) + }) { + pending.truncate(cut); + hit_cursor_boundary = true; + } + } + + // ctx.max_items: clamp the dedup'd batch to the remaining budget before ingest. + cap.clamp_batch(&mut pending); + let ingestor = MemoryTreeIngestor { config: ctx.config.as_ref(), connection_id: &connection_id, @@ -341,6 +389,13 @@ impl ComposioProvider for ClickUpProvider { state.mark_synced(key); } total_persisted += outcome.persisted; + cap.record(outcome.persisted); + + // ctx.max_items precise cap: once the per-source cap is hit, stop paginating. + if cap.is_reached() { + hit_cap_boundary = true; + break 'workspaces; + } if hit_cursor_boundary { tracing::debug!( @@ -351,6 +406,18 @@ impl ComposioProvider for ClickUpProvider { break; } + // ctx.max_items hard stop. + if cap.is_reached() { + tracing::debug!( + workspace_id = %workspace_id, + page = page_num, + total_persisted, + "[composio:clickup] [memory_sync] max_items reached, stopping pagination" + ); + hit_cap_boundary = true; + break 'workspaces; + } + // ClickUp's filtered-team-tasks endpoint signals the last // page implicitly: when fewer than `page_size` results // come back, there are no more pages. @@ -367,8 +434,17 @@ impl ComposioProvider for ClickUpProvider { } // ── Step 6: advance cursor and save state ─────────────────── - if let Some(new_cursor) = newest_updated { - state.advance_cursor(&new_cursor); + // Hold the cursor on a cap-truncated pass so the next sync re-scans the unseen tail. + if !hit_cap_boundary { + if let Some(new_cursor) = newest_updated { + state.advance_cursor(&new_cursor); + } + } else { + tracing::warn!( + connection_id = %connection_id, + "[composio:clickup] holding cursor — cap-truncated pass; next sync will re-scan \ + the unseen tail" + ); } state.set_last_sync_at_ms(sync::now_ms()); state.save(&memory).await?; diff --git a/src/openhuman/memory_sync/composio/providers/github/provider.rs b/src/openhuman/memory_sync/composio/providers/github/provider.rs index db7996eb8..42749b8c8 100644 --- a/src/openhuman/memory_sync/composio/providers/github/provider.rs +++ b/src/openhuman/memory_sync/composio/providers/github/provider.rs @@ -200,8 +200,42 @@ impl ComposioProvider for GitHubProvider { _ => PAGE_SIZE, }; - // Build the base search query. - let query = build_search_query(&login, state.cursor.as_deref()); + // ctx.max_items: route through ItemCap — page ceiling, mid-page + // per-item break, and post-page hard stop all share one source of truth. + let mut cap = super::super::helpers::ItemCap::new(ctx.max_items); + let effective_max_pages = cap.max_pages(page_size, MAX_PAGES); + if ctx.max_items.is_some() && effective_max_pages < MAX_PAGES { + tracing::debug!( + connection_id = %connection_id, + max_items = ?ctx.max_items, + effective_max_pages, + "[composio:github] [memory_sync] applying max_items page cap" + ); + } + + // ctx.sync_depth_days: inject `updated:>{date}` on first sync when set. + let depth_query_fragment: Option = if state.cursor.is_none() { + ctx.sync_depth_days.map(|days| { + let floor = chrono::Utc::now() - chrono::Duration::days(days as i64); + let s = floor.format("%Y-%m-%dT%H:%M:%SZ").to_string(); + tracing::debug!( + connection_id = %connection_id, + sync_depth_days = days, + floor = %s, + "[composio:github] [memory_sync] injecting updated:> date filter on first sync" + ); + format!("updated:>{s}") + }) + } else { + None + }; + + // Build the base search query — include depth fragment when set. + let query = build_search_query_with_depth( + &login, + state.cursor.as_deref(), + depth_query_fragment.as_deref(), + ); let mut total_fetched: usize = 0; let mut total_persisted: usize = 0; @@ -214,8 +248,9 @@ impl ComposioProvider for GitHubProvider { // until its next edit. Already-synced items are skipped cheaply via // `is_synced` on the re-fetch, so the cost of holding is minimal. let mut had_ingest_failures = false; + let mut hit_cap_boundary = false; - 'pages: for page_num in 1..=MAX_PAGES { + 'pages: for page_num in 1..=effective_max_pages { if state.budget_exhausted() { tracing::info!( page = page_num, @@ -335,6 +370,7 @@ impl ComposioProvider for GitHubProvider { Ok(_chunks_written) => { state.mark_synced(&sync_key); total_persisted += 1; + cap.record(1); } Err(e) => { had_ingest_failures = true; @@ -345,6 +381,24 @@ impl ComposioProvider for GitHubProvider { ); } } + + // ctx.max_items precise cap: stop mid-page so we never persist + // more than the cap even when a single page exceeds it. + if cap.is_reached() { + hit_cap_boundary = true; + break; + } + } + + // ctx.max_items hard stop. + if cap.is_reached() { + tracing::debug!( + page = page_num, + total_persisted, + "[composio:github] [memory_sync] max_items reached, stopping pagination" + ); + hit_cap_boundary = true; + break; } // GitHub search pages are 0-indexed in terms of total results; @@ -366,15 +420,18 @@ impl ComposioProvider for GitHubProvider { // the delete-first memory-tree pipeline (#2885). `set_last_sync_at_ms` // still advances — that's just a heartbeat, not a fetch-window // boundary, so it's safe to record that we did attempt a sync. - if !had_ingest_failures { + // Hold the cursor on a cap-truncated pass so the next sync re-scans the unseen tail. + if !had_ingest_failures && !hit_cap_boundary { if let Some(new_cursor) = newest_updated { state.advance_cursor(&new_cursor); } } else { tracing::warn!( connection_id = %connection_id, - "[composio:github] holding cursor — ingest failures this pass; next sync will \ - re-fetch the failed range" + had_ingest_failures, + hit_cap_boundary, + "[composio:github] holding cursor — ingest failures or cap-truncated pass; next \ + sync will re-fetch the failed range" ); } state.set_last_sync_at_ms(sync::now_ms()); @@ -896,3 +953,48 @@ pub(super) fn build_search_query(login: &str, cursor: Option<&str>) -> String { None => format!("involves:{login}"), } } + +/// Extended variant that optionally appends a `sync_depth_days` fragment on +/// first sync (no cursor). The `depth_fragment` is expected to be a pre-built +/// `"updated:>{date}"` string. +pub(super) fn build_search_query_with_depth( + login: &str, + cursor: Option<&str>, + depth_fragment: Option<&str>, +) -> String { + match cursor { + Some(c) => format!("involves:{login} updated:>{c}"), + None => match depth_fragment { + Some(fragment) => format!("involves:{login} {fragment}"), + None => format!("involves:{login}"), + }, + } +} + +#[cfg(test)] +mod depth_tests { + use super::*; + + #[test] + fn build_search_query_with_depth_no_cursor_no_depth() { + let q = build_search_query_with_depth("alice", None, None); + assert_eq!(q, "involves:alice"); + } + + #[test] + fn build_search_query_with_depth_no_cursor_with_depth() { + let q = build_search_query_with_depth("alice", None, Some("updated:>2024-01-01T00:00:00Z")); + assert_eq!(q, "involves:alice updated:>2024-01-01T00:00:00Z"); + } + + #[test] + fn build_search_query_with_depth_cursor_wins_over_depth() { + // When cursor is set, depth fragment is ignored. + let q = build_search_query_with_depth( + "alice", + Some("2024-06-01T00:00:00Z"), + Some("updated:>2024-01-01T00:00:00Z"), + ); + assert_eq!(q, "involves:alice updated:>2024-06-01T00:00:00Z"); + } +} diff --git a/src/openhuman/memory_sync/composio/providers/gmail/provider.rs b/src/openhuman/memory_sync/composio/providers/gmail/provider.rs index cad580abe..bcc57199a 100644 --- a/src/openhuman/memory_sync/composio/providers/gmail/provider.rs +++ b/src/openhuman/memory_sync/composio/providers/gmail/provider.rs @@ -258,7 +258,7 @@ impl ComposioProvider for GmailProvider { // legitimately want the larger ceiling — and the cap only // kicks in when we have a prior `last_sync_at_ms` to compare // against, so first-ever syncs are unaffected. - let max_pages = match reason { + let base_max_pages = match reason { SyncReason::ConnectionCreated => MAX_PAGES_PER_SYNC, _ => match state.last_sync_at_ms { Some(last_ms) if sync::now_ms().saturating_sub(last_ms) < RECENT_SYNC_WINDOW_MS => { @@ -274,6 +274,36 @@ impl ComposioProvider for GmailProvider { }, }; + // ctx.max_items: route through ItemCap so the page ceiling, mid-page + // clamp, and post-page hard stop all share one source of truth. + let mut cap = super::super::helpers::ItemCap::new(ctx.max_items); + let max_pages = cap.max_pages(page_size, base_max_pages); + if ctx.max_items.is_some() && max_pages < base_max_pages { + tracing::debug!( + connection_id = %connection_id, + max_items = ?ctx.max_items, + page_size, + effective_max_pages = max_pages, + "[composio:gmail] [memory_sync] applying max_items page cap from source config" + ); + } + + // ctx.sync_depth_days: on first sync (no cursor), add an after: floor. + let depth_floor_filter: Option = if state.cursor.is_none() { + ctx.sync_depth_days.map(|days| { + let floor_secs = super::super::helpers::epoch_floor_from_depth(days); + tracing::debug!( + connection_id = %connection_id, + sync_depth_days = days, + floor_epoch_secs = floor_secs, + "[composio:gmail] [memory_sync] applying sync_depth_days floor on first sync" + ); + floor_secs.to_string() + }) + } else { + None + }; + let mut total_fetched: usize = 0; let mut total_persisted: usize = 0; let mut total_requests: u32 = 0; @@ -281,6 +311,7 @@ impl ComposioProvider for GmailProvider { let mut newest_id: Option = None; let mut page_token: Option = None; let mut stop_reason: &'static str = "max_pages"; + let mut hit_cap_boundary = false; for page_num in 0..max_pages { if state.budget_exhausted() { @@ -321,6 +352,9 @@ impl ComposioProvider for GmailProvider { "[composio:gmail] using day-level filter from cursor (epoch parse failed)" ); } + } else if let Some(ref floor) = depth_floor_filter { + // First sync with sync_depth_days: apply the epoch floor. + query.push_str(&format!(" after:{floor}")); } let mut args = json!({ @@ -453,6 +487,11 @@ impl ComposioProvider for GmailProvider { new_messages.push(msg.clone()); } + // ctx.max_items precise cap: clamp the per-page batch before ingest + // so a single page larger than the budget is never over-persisted. + cap.clamp_batch(&mut new_messages); + cap.clamp_batch(&mut pending_synced_ids); + // Single batched ingest into memory_tree. Chunk IDs are // content-hashed so re-ingest of the same message is an // idempotent UPSERT at the SQL layer; per-message dedup above @@ -483,6 +522,7 @@ impl ComposioProvider for GmailProvider { // persist path. n is the chunk count which we log // for diagnostic purposes only. total_persisted += new_messages.len(); + cap.record(new_messages.len()); tracing::debug!( page = page_num, new_messages = new_messages.len(), @@ -512,6 +552,18 @@ impl ComposioProvider for GmailProvider { break; } + // ctx.max_items hard stop: break once the per-source cap is reached. + if cap.is_reached() { + tracing::debug!( + page = page_num, + total_persisted, + "[composio:gmail] [memory_sync] max_items reached, stopping pagination" + ); + stop_reason = "max_items"; + hit_cap_boundary = true; + break; + } + // Check for next page token. page_token = sync::extract_page_token(&resp.data); if page_token.is_none() { @@ -522,8 +574,17 @@ impl ComposioProvider for GmailProvider { } // ── Step 5: advance cursor and save state ─────────────────── - if let Some(new_cursor) = newest_date { - state.advance_cursor(&new_cursor); + // Hold the cursor on a cap-truncated pass so the next sync re-scans the unseen tail. + if !hit_cap_boundary { + if let Some(new_cursor) = newest_date { + state.advance_cursor(&new_cursor); + } + } else { + tracing::warn!( + connection_id = %connection_id, + "[composio:gmail] holding cursor — cap-truncated pass; next sync will re-scan \ + the unseen tail" + ); } if let Some(ref freshest) = newest_id { state.set_last_seen_id(freshest); @@ -616,3 +677,7 @@ impl ComposioProvider for GmailProvider { Ok(()) } } + +// Cap/date-floor math lives in the shared `super::super::helpers` module +// (`ItemCap`, `pages_for_max_items`, `epoch_floor_from_depth`) so every provider +// shares one implementation — see that module for the unit tests. diff --git a/src/openhuman/memory_sync/composio/providers/helpers.rs b/src/openhuman/memory_sync/composio/providers/helpers.rs index 8d2432802..d326689e0 100644 --- a/src/openhuman/memory_sync/composio/providers/helpers.rs +++ b/src/openhuman/memory_sync/composio/providers/helpers.rs @@ -46,6 +46,178 @@ pub(crate) fn merge_extra(args: &mut serde_json::Value, extra: &serde_json::Valu } } +// ── Cap enforcement helpers ────────────────────────────────────────────── + +/// Compute the number of API pages needed to cover `max_items` at `page_size` +/// items per page, rounding up. +/// +/// Returns `u32::MAX` when `page_size == 0` to avoid division by zero; +/// callers should treat this as "no page cap beyond the provider's own upper +/// bound". +pub(crate) fn pages_for_max_items(max_items: u32, page_size: u32) -> u32 { + if page_size == 0 { + return u32::MAX; + } + // Widen to u64 before the addition to prevent overflow for large cap values. + (((max_items as u64) + (page_size as u64) - 1) / (page_size as u64)).min(u32::MAX as u64) as u32 +} + +/// Compute the Unix epoch timestamp (seconds) for `sync_depth_days` days ago. +/// Used to build after-date filters (e.g. Gmail `after:`) on first sync. +pub(crate) fn epoch_floor_from_depth(sync_depth_days: u32) -> i64 { + let now = chrono::Utc::now(); + let floor = now - chrono::Duration::days(sync_depth_days as i64); + floor.timestamp() +} + +/// Single source of truth for the per-sync `max_items` cap. +/// +/// Every Composio provider used to open-code three near-identical blocks — a +/// page-count cap, a mid-page clamp, and a post-page hard stop — which is how +/// the same off-by-a-page bug ended up in five providers and was missed in a +/// sixth. Funnelling all of them through this one type keeps the rule in one +/// place: construct it from `ctx.max_items`, derive the page cap, clamp each +/// batch (or check per item), and stop once the cap is reached. +/// +/// `None` cap means "no item limit beyond the provider's own internal page +/// ceiling" (e.g. after the user clicks "All In"). +#[derive(Debug, Clone, Copy)] +pub(crate) struct ItemCap { + cap: Option, + persisted: usize, +} + +impl ItemCap { + /// Build from a source's `max_items` value (`None` = uncapped). + pub(crate) fn new(max_items: Option) -> Self { + Self { + cap: max_items.map(|n| n as usize), + persisted: 0, + } + } + + /// The page ceiling to actually fetch: the smaller of the provider's own + /// `fallback` (e.g. `MAX_PAGES_PER_SYNC`) and the pages needed to cover the + /// cap. Uncapped → `fallback` unchanged. + pub(crate) fn max_pages(&self, page_size: u32, fallback: u32) -> u32 { + match self.cap { + Some(cap) => pages_for_max_items(cap as u32, page_size).min(fallback), + None => fallback, + } + } + + /// How many more items may still be persisted. `None` = unlimited. + pub(crate) fn remaining(&self) -> Option { + self.cap.map(|cap| cap.saturating_sub(self.persisted)) + } + + /// True once the cap is set and reached — callers `break` their pagination. + pub(crate) fn is_reached(&self) -> bool { + matches!(self.remaining(), Some(0)) + } + + /// Record `n` newly-persisted items against the budget. + pub(crate) fn record(&mut self, n: usize) { + self.persisted = self.persisted.saturating_add(n); + } + + /// Truncate a to-ingest batch down to the remaining budget, so a single + /// page larger than the cap never over-persists. No-op when uncapped. + pub(crate) fn clamp_batch(&self, batch: &mut Vec) { + if let Some(remaining) = self.remaining() { + if batch.len() > remaining { + batch.truncate(remaining); + } + } + } +} + +#[cfg(test)] +mod cap_helper_tests { + use super::*; + + #[test] + fn pages_for_max_items_rounds_up() { + assert_eq!(pages_for_max_items(100, 25), 4); + assert_eq!(pages_for_max_items(101, 25), 5); + assert_eq!(pages_for_max_items(1, 25), 1); + assert_eq!(pages_for_max_items(50, 50), 1); + assert_eq!(pages_for_max_items(51, 50), 2); + } + + #[test] + fn pages_for_max_items_zero_page_size() { + assert_eq!(pages_for_max_items(100, 0), u32::MAX); + } + + #[test] + fn epoch_floor_from_depth_is_in_the_past() { + let floor = epoch_floor_from_depth(30); + let now = chrono::Utc::now().timestamp(); + assert!(floor < now); + let diff_days = (now - floor) / 86400; + assert!( + diff_days >= 29 && diff_days <= 31, + "expected ~30 days in past, got {diff_days}" + ); + } + + #[test] + fn item_cap_uncapped_is_never_reached() { + let mut cap = ItemCap::new(None); + assert_eq!(cap.remaining(), None); + assert!(!cap.is_reached()); + cap.record(1_000_000); + assert!(!cap.is_reached()); + assert_eq!( + cap.max_pages(25, 20), + 20, + "uncapped keeps the provider fallback" + ); + } + + #[test] + fn item_cap_tracks_remaining_and_reached() { + let mut cap = ItemCap::new(Some(3)); + assert_eq!(cap.remaining(), Some(3)); + assert!(!cap.is_reached()); + cap.record(2); + assert_eq!(cap.remaining(), Some(1)); + assert!(!cap.is_reached()); + cap.record(5); // saturates, never underflows + assert_eq!(cap.remaining(), Some(0)); + assert!(cap.is_reached()); + } + + #[test] + fn item_cap_max_pages_is_min_of_fallback_and_needed() { + // cap=2, page_size=50 → 1 page needed, well under the fallback. + assert_eq!(ItemCap::new(Some(2)).max_pages(50, 20), 1); + // cap=1000, page_size=25 → 40 pages needed, clamped to fallback 20. + assert_eq!(ItemCap::new(Some(1000)).max_pages(25, 20), 20); + } + + #[test] + fn item_cap_clamp_batch_truncates_to_remaining() { + let cap = ItemCap::new(Some(2)); + let mut batch = vec![1, 2, 3, 4, 5]; + cap.clamp_batch(&mut batch); + assert_eq!(batch, vec![1, 2]); + + // Uncapped leaves the batch untouched. + let mut full = vec![1, 2, 3]; + ItemCap::new(None).clamp_batch(&mut full); + assert_eq!(full, vec![1, 2, 3]); + + // After recording progress, clamp uses the reduced budget. + let mut cap2 = ItemCap::new(Some(5)); + cap2.record(3); + let mut batch2 = vec![1, 2, 3, 4]; + cap2.clamp_batch(&mut batch2); + assert_eq!(batch2, vec![1, 2], "only 2 of the 5 budget remained"); + } +} + /// Resolve the first array found among `array_paths` (dotted object /// paths), then return the first non-empty string at one of `fields` /// on that array's first element. Complements [`pick_str`], which diff --git a/src/openhuman/memory_sync/composio/providers/linear/provider.rs b/src/openhuman/memory_sync/composio/providers/linear/provider.rs index eaa5ab478..5df1c9a52 100644 --- a/src/openhuman/memory_sync/composio/providers/linear/provider.rs +++ b/src/openhuman/memory_sync/composio/providers/linear/provider.rs @@ -198,14 +198,41 @@ impl ComposioProvider for LinearProvider { _ => PAGE_SIZE, }; + // ctx.max_items: route through ItemCap — page ceiling, mid-page + // per-item break, and post-page hard stop all share one source of truth. + let mut cap = super::super::helpers::ItemCap::new(ctx.max_items); + let effective_max_pages = cap.max_pages(page_size as u32, MAX_PAGES_PER_SYNC); + if ctx.max_items.is_some() && effective_max_pages < MAX_PAGES_PER_SYNC { + tracing::debug!( + connection_id = %connection_id, + max_items = ?ctx.max_items, + effective_max_pages, + "[composio:linear] [memory_sync] applying max_items page cap" + ); + } + + // ctx.sync_depth_days: oldest allowed updatedAt for client-side skip. + let oldest_allowed_time: Option = ctx.sync_depth_days.map(|days| { + let floor = chrono::Utc::now() - chrono::Duration::days(days as i64); + let s = floor.to_rfc3339(); + tracing::debug!( + connection_id = %connection_id, + sync_depth_days = days, + oldest_allowed = %s, + "[composio:linear] [memory_sync] applying sync_depth_days floor" + ); + s + }); + let mut total_fetched: usize = 0; let mut total_persisted: usize = 0; let mut had_persist_failures = false; let mut newest_updated: Option = None; let mut after_cursor: Option = None; let mut hit_cursor_boundary = false; + let mut hit_cap_boundary = false; - for page_num in 0..MAX_PAGES_PER_SYNC { + for page_num in 0..effective_max_pages { if state.budget_exhausted() { tracing::info!( page = page_num, @@ -256,11 +283,30 @@ impl ComposioProvider for LinearProvider { } // ── Per-item dedup + bounded-concurrency ingest ────────── - let (pending, page_hit_boundary) = select_pending(&issues, &state, &mut newest_updated); + let (mut pending, page_hit_boundary) = + select_pending(&issues, &state, &mut newest_updated); if page_hit_boundary { hit_cursor_boundary = true; } + // ctx.sync_depth_days: drop items updated before the depth floor. `pending` is + // in descending timestamp order, so truncate at the first item below the floor + // and signal cursor-boundary so pagination stops. + if let Some(ref floor) = oldest_allowed_time { + if let Some(cut) = pending.iter().position(|p| { + p.updated + .as_deref() + .map(|t| t < floor.as_str()) + .unwrap_or(false) + }) { + pending.truncate(cut); + hit_cursor_boundary = true; + } + } + + // ctx.max_items: clamp the dedup'd batch to the remaining budget before ingest. + cap.clamp_batch(&mut pending); + let ingestor = MemoryTreeIngestor { config: ctx.config.as_ref(), connection_id: &connection_id, @@ -270,10 +316,17 @@ impl ComposioProvider for LinearProvider { state.mark_synced(key); } total_persisted += outcome.persisted; + cap.record(outcome.persisted); if outcome.had_failures { had_persist_failures = true; } + // ctx.max_items precise cap: once the per-source cap is hit, stop paginating. + if cap.is_reached() { + hit_cap_boundary = true; + break; + } + if hit_cursor_boundary { tracing::debug!( page = page_num, @@ -282,6 +335,17 @@ impl ComposioProvider for LinearProvider { break; } + // ctx.max_items hard stop. + if cap.is_reached() { + tracing::debug!( + page = page_num, + total_persisted, + "[composio:linear] [memory_sync] max_items reached, stopping pagination" + ); + hit_cap_boundary = true; + break; + } + // Advance to the next page using Linear's cursor-based pagination. match sync::extract_pagination_cursor(&resp.data) { Some(next_cursor) => { @@ -298,10 +362,17 @@ impl ComposioProvider for LinearProvider { } // ── Step 5: advance cursor and save state ──────────────────── + // Hold the cursor on a cap-truncated pass so the next sync re-scans the unseen tail. if had_persist_failures { tracing::warn!( "[composio:linear] persist failures seen; keeping previous cursor for retry" ); + } else if hit_cap_boundary { + tracing::warn!( + hit_cap_boundary, + "[composio:linear] holding cursor — cap-truncated pass; next sync will re-scan \ + the unseen tail" + ); } else if let Some(new_cursor) = newest_updated { state.advance_cursor(&new_cursor); } diff --git a/src/openhuman/memory_sync/composio/providers/notion/provider.rs b/src/openhuman/memory_sync/composio/providers/notion/provider.rs index ee7e4ea2d..075cd799f 100644 --- a/src/openhuman/memory_sync/composio/providers/notion/provider.rs +++ b/src/openhuman/memory_sync/composio/providers/notion/provider.rs @@ -186,6 +186,33 @@ impl ComposioProvider for NotionProvider { _ => PAGE_SIZE, }; + // ctx.max_items: route through ItemCap — page ceiling, mid-page + // per-item break, and post-page hard stop all share one source of truth. + let mut cap = super::super::helpers::ItemCap::new(ctx.max_items); + let effective_max_pages = cap.max_pages(page_size, MAX_PAGES_PER_SYNC); + if ctx.max_items.is_some() && effective_max_pages < MAX_PAGES_PER_SYNC { + tracing::debug!( + connection_id = %connection_id, + max_items = ?ctx.max_items, + effective_max_pages, + "[composio:notion] [memory_sync] applying max_items page cap" + ); + } + + // ctx.sync_depth_days: compute the oldest allowed edited_time string + // for client-side skipping of stale results. + let oldest_allowed_time: Option = ctx.sync_depth_days.map(|days| { + let floor = chrono::Utc::now() - chrono::Duration::days(days as i64); + let s = floor.to_rfc3339(); + tracing::debug!( + connection_id = %connection_id, + sync_depth_days = days, + oldest_allowed = %s, + "[composio:notion] [memory_sync] applying sync_depth_days floor" + ); + s + }); + let mut total_fetched: usize = 0; let mut total_persisted: usize = 0; let mut newest_edited_time: Option = None; @@ -198,8 +225,9 @@ impl ComposioProvider for NotionProvider { // its next edit. Already-synced items are skipped cheaply via // `is_synced` on the re-fetch, so the cost of holding is minimal. let mut had_ingest_failures = false; + let mut hit_cap_boundary = false; - for page_num in 0..MAX_PAGES_PER_SYNC { + for page_num in 0..effective_max_pages { if state.budget_exhausted() { tracing::info!( page = page_num, @@ -249,9 +277,27 @@ impl ComposioProvider for NotionProvider { } // ── Step 4a: dedupe + decide which pages to ingest ────── - let (pending, hit_cursor_boundary) = + let (mut pending, mut hit_cursor_boundary) = select_pending(&results, &state, &mut newest_edited_time); + // ctx.sync_depth_days: drop items edited before the depth floor. `pending` is + // in descending timestamp order, so truncate at the first item below the floor + // and signal cursor-boundary so pagination stops. + if let Some(ref floor) = oldest_allowed_time { + if let Some(cut) = pending.iter().position(|p| { + p.edited_time + .as_deref() + .map(|t| t < floor.as_str()) + .unwrap_or(false) + }) { + pending.truncate(cut); + hit_cursor_boundary = true; + } + } + + // ctx.max_items: clamp the dedup'd batch to the remaining budget before ingest. + cap.clamp_batch(&mut pending); + // ── Step 4b: ingest queued pages (bounded concurrency) ── let ingestor = MemoryTreeIngestor { config: ctx.config.as_ref(), @@ -262,10 +308,17 @@ impl ComposioProvider for NotionProvider { state.mark_synced(key); } total_persisted += outcome.persisted; + cap.record(outcome.persisted); if outcome.had_failures { had_ingest_failures = true; } + // ctx.max_items precise cap: once the per-source cap is hit, stop paginating. + if cap.is_reached() { + hit_cap_boundary = true; + break; + } + if hit_cursor_boundary { tracing::debug!( page = page_num, @@ -274,6 +327,17 @@ impl ComposioProvider for NotionProvider { break; } + // ctx.max_items hard stop. + if cap.is_reached() { + tracing::debug!( + page = page_num, + total_persisted, + "[composio:notion] [memory_sync] max_items reached, stopping pagination" + ); + hit_cap_boundary = true; + break; + } + // Check for next page cursor from Notion API. notion_cursor = sync::extract_notion_cursor(&resp.data); if notion_cursor.is_none() { @@ -287,15 +351,18 @@ impl ComposioProvider for NotionProvider { // Hold the cursor when any item failed to ingest this pass. See the // `had_ingest_failures` declaration above for why this matters under // the delete-first memory-tree pipeline (#2885). - if !had_ingest_failures { + // Hold the cursor on a cap-truncated pass so the next sync re-scans the unseen tail. + if !had_ingest_failures && !hit_cap_boundary { if let Some(new_cursor) = newest_edited_time { state.advance_cursor(&new_cursor); } } else { tracing::warn!( connection_id = %connection_id, - "[composio:notion] holding cursor — ingest failures this pass; next sync will \ - re-fetch the failed range" + had_ingest_failures, + hit_cap_boundary, + "[composio:notion] holding cursor — ingest failures or cap-truncated pass; next \ + sync will re-fetch the failed range" ); } state.save(&memory).await?; diff --git a/src/openhuman/memory_sync/composio/providers/slack/provider.rs b/src/openhuman/memory_sync/composio/providers/slack/provider.rs index 7ca8d4403..6cdd8dea6 100644 --- a/src/openhuman/memory_sync/composio/providers/slack/provider.rs +++ b/src/openhuman/memory_sync/composio/providers/slack/provider.rs @@ -468,6 +468,13 @@ impl ComposioProvider for SlackProvider { let mut total_messages_ingested: usize = 0; let mut channels_processed: usize = 0; let mut channels_errored: usize = 0; + let mut hit_cap_boundary = false; + + // ctx.max_items: ItemCap is threaded through process_channel so the + // per-page batch is clamped before ingest and the channel loop stops + // precisely at the cap — the old coarse post-channel check allowed a + // single page/channel to blow past the cap. + let mut cap = super::super::helpers::ItemCap::new(ctx.max_items); // 2. Per-channel: fetch → post-process → enrich → ingest. for channel in &channels { @@ -488,6 +495,7 @@ impl ComposioProvider for SlackProvider { now, &users, &connection_id, + &mut cap, ) .await { @@ -506,6 +514,27 @@ impl ComposioProvider for SlackProvider { } } + // ctx.max_items hard stop across all channels (precise — cap was + // already applied inside process_channel so this break fires + // exactly when the budget is exhausted, not one channel later). + if cap.is_reached() { + // Hold the cursor on a cap-truncated pass so the next sync re-scans the unseen tail. + hit_cap_boundary = true; + tracing::debug!( + connection_id = %connection_id, + total_messages_ingested, + "[composio:slack] [memory_sync] max_items reached, stopping channel iteration" + ); + // Save state before breaking without advancing the cursor. + if let Err(err) = state.save(&memory).await { + tracing::warn!( + error = %err, + "[composio:slack] state save failed after cap-stop (non-fatal)" + ); + } + break; + } + state.advance_cursor(sync::encode_cursors(&cursors)); if let Err(err) = state.save(&memory).await { tracing::warn!( @@ -515,6 +544,15 @@ impl ComposioProvider for SlackProvider { } } + if hit_cap_boundary { + // Hold the cursor on a cap-truncated pass so the next sync re-scans the unseen tail. + tracing::warn!( + connection_id = %connection_id, + "[composio:slack] cap-truncated pass; cursor held so next sync re-scans the \ + unseen tail" + ); + } + let finished_at_ms = sync::now_ms(); let summary = format!( "slack sync: channels_processed={channels_processed} \ @@ -608,7 +646,12 @@ async fn list_all_channels( } /// Pull one channel's history since its cursor, post-process + enrich each -/// page, then ingest all messages. Returns the number of chunks written. +/// page, then ingest all messages. Returns the number of messages written. +/// +/// `cap` is the shared [`super::super::helpers::ItemCap`] for the sync pass. +/// Each page's message batch is clamped to the remaining budget before ingest +/// so the per-sync `max_items` limit is respected precisely regardless of how +/// many messages a single channel/page returns. async fn process_channel( ctx: &ProviderContext, state: &mut SyncState, @@ -617,13 +660,26 @@ async fn process_channel( now: chrono::DateTime, users: &SlackUsers, connection_id: &str, + cap: &mut super::super::helpers::ItemCap, ) -> Result { // Cursor value is a raw Slack `ts` (`"."`) preserved // with full precision, so multi-message-per-second channels don't // replay the whole second on the next incremental fetch. When no // cursor exists yet, fall back to `.000000`. + // ctx.sync_depth_days wins over the env-var OPENHUMAN_SLACK_BACKFILL_DAYS + // default when set — it comes from the user-configured source entry. let oldest_ts = cursors.get(&channel.id).cloned().unwrap_or_else(|| { - let secs = (now - chrono::Duration::days(backfill_days())).timestamp(); + let depth_days = ctx + .sync_depth_days + .map(|d| d as i64) + .unwrap_or_else(backfill_days); + let secs = (now - chrono::Duration::days(depth_days)).timestamp(); + tracing::debug!( + channel = %channel.id, + depth_days, + oldest_ts_secs = secs, + "[composio:slack] [memory_sync] computing oldest_ts for backfill" + ); format!("{secs}.000000") }); @@ -676,6 +732,23 @@ async fn process_channel( break; } all_messages.extend(msgs); + + // Stop fetching further pages for this channel if we have already + // accumulated enough to fill the remaining budget (checked against + // remaining() which accounts for items recorded by previous channels). + if let Some(remaining) = cap.remaining() { + if all_messages.len() >= remaining { + tracing::debug!( + channel = %channel.id, + page = page_num, + accumulated = all_messages.len(), + remaining, + "[composio:slack] [memory_sync] budget nearly full, stopping history pagination" + ); + break; + } + } + cursor = sync::extract_next_cursor(&resp.data); if cursor.is_none() { break; @@ -690,6 +763,19 @@ async fn process_channel( return Ok(0); } + // ctx.max_items precise cap: clamp the full accumulated batch to the + // remaining budget before ingest so we never persist more than the cap + // allows, even if a single channel/page returned more than what remains. + cap.clamp_batch(&mut all_messages); + + if all_messages.is_empty() { + tracing::debug!( + channel = %channel.id, + "[composio:slack] [memory_sync] cap already reached, skipping channel ingest" + ); + return Ok(0); + } + let msg_count = all_messages.len(); tracing::info!( channel = %channel.id, @@ -713,13 +799,16 @@ async fn process_channel( { cursors.insert(channel.id.clone(), latest); } + cap.record(msg_count); tracing::info!( channel = %channel.id, messages = msg_count, chunks, "[composio:slack] channel ingest done" ); - Ok(chunks) + // Return message count (consistent with the sync path which + // counts messages, not chunks, for the items_ingested metric). + Ok(msg_count) } Err(e) => { tracing::warn!( diff --git a/src/openhuman/memory_sync/composio/providers/slack/rpc.rs b/src/openhuman/memory_sync/composio/providers/slack/rpc.rs index f0d9a147b..9d4fe23f2 100644 --- a/src/openhuman/memory_sync/composio/providers/slack/rpc.rs +++ b/src/openhuman/memory_sync/composio/providers/slack/rpc.rs @@ -111,6 +111,8 @@ pub async fn sync_trigger_rpc( toolkit: conn.toolkit.clone(), connection_id: Some(conn.id.clone()), usage: Default::default(), + max_items: None, + sync_depth_days: None, }; match provider.sync(&ctx, SyncReason::Manual).await { Ok(o) => outcomes.push(o), diff --git a/src/openhuman/memory_sync/composio/providers/types.rs b/src/openhuman/memory_sync/composio/providers/types.rs index ecaa503d6..ece7a5430 100644 --- a/src/openhuman/memory_sync/composio/providers/types.rs +++ b/src/openhuman/memory_sync/composio/providers/types.rs @@ -295,6 +295,18 @@ pub struct ProviderContext { /// (`run_connection_sync`) reads it back. Non-sync callers (agent tools, /// task-source fetches) leave it at zero — harmless. pub usage: ComposioUsageHandle, + /// Maximum items to fetch in a single sync pass. + /// + /// Set from the corresponding `MemorySourceEntry.max_items` field at + /// sync-dispatch time. `None` means no cap beyond the provider's own + /// internal upper bounds. + pub max_items: Option, + /// Maximum sync depth window in days. + /// + /// Set from `MemorySourceEntry.sync_depth_days`. When `Some(n)`, the + /// provider only fetches items from the last `n` days. `None` means + /// no additional depth restriction beyond the provider's cursor. + pub sync_depth_days: Option, } impl ProviderContext { @@ -325,6 +337,8 @@ impl ProviderContext { toolkit: toolkit.into(), connection_id, usage: ComposioUsageHandle::default(), + max_items: None, + sync_depth_days: None, }), Err(e) => { tracing::debug!( @@ -487,6 +501,8 @@ mod tests { toolkit: "gmail".to_string(), connection_id: None, usage: ComposioUsageHandle::default(), + max_items: None, + sync_depth_days: None, }; let cloned = ctx.clone(); @@ -542,6 +558,8 @@ mod tests { toolkit: "gmail".to_string(), connection_id: None, usage: ComposioUsageHandle::default(), + max_items: None, + sync_depth_days: None, }; let res = ctx.execute("GMAIL_FETCH_EMAILS", None).await; // The actual HTTP call will fail in the unit-test sandbox, but @@ -575,6 +593,8 @@ mod tests { toolkit: "gmail".to_string(), connection_id: None, usage: ComposioUsageHandle::default(), + max_items: None, + sync_depth_days: None, }; let res = ctx.execute("GMAIL_FETCH_EMAILS", None).await; let err = res.expect_err("no backend session must error"); diff --git a/src/openhuman/task_sources/ops.rs b/src/openhuman/task_sources/ops.rs index 159893b57..13142552c 100644 --- a/src/openhuman/task_sources/ops.rs +++ b/src/openhuman/task_sources/ops.rs @@ -153,6 +153,8 @@ pub async fn preview_filter( toolkit: provider.as_str().to_string(), connection_id: connection_id.filter(|s| !s.trim().is_empty()), usage: Default::default(), + max_items: None, + sync_depth_days: None, }; let max = max.unwrap_or(config.task_sources.max_tasks_per_fetch); let fetch_filter = filter::to_fetch_filter(&filter_spec, max); @@ -179,6 +181,8 @@ pub async fn list_databases( toolkit: provider.as_str().to_string(), connection_id: connection_id.filter(|s| !s.trim().is_empty()), usage: Default::default(), + max_items: None, + sync_depth_days: None, }; let databases = provider_impl .list_databases(&ctx) diff --git a/src/openhuman/task_sources/pipeline.rs b/src/openhuman/task_sources/pipeline.rs index 9094bdc59..74953f1e2 100644 --- a/src/openhuman/task_sources/pipeline.rs +++ b/src/openhuman/task_sources/pipeline.rs @@ -102,6 +102,8 @@ async fn run_inner( toolkit: source.provider.as_str().to_string(), connection_id: source.connection_id.clone(), usage: Default::default(), + max_items: None, + sync_depth_days: None, }; let fetch_filter = filter::to_fetch_filter(&source.filter, source.max_tasks_per_fetch); diff --git a/tests/config_auth_app_state_connectivity_e2e.rs b/tests/config_auth_app_state_connectivity_e2e.rs index 865b2deec..9d0de1b3a 100644 --- a/tests/config_auth_app_state_connectivity_e2e.rs +++ b/tests/config_auth_app_state_connectivity_e2e.rs @@ -2856,6 +2856,7 @@ async fn worker_a_controller_schemas_are_fully_exposed() { "memory_sources", vec![ "openhuman.memory_sources_add", + "openhuman.memory_sources_apply_all_in", "openhuman.memory_sources_estimate_sync_cost", "openhuman.memory_sources_get", "openhuman.memory_sources_list", diff --git a/tests/memory_sync_providers_raw_coverage_e2e.rs b/tests/memory_sync_providers_raw_coverage_e2e.rs index d0e9388fc..fa7d0c57b 100644 --- a/tests/memory_sync_providers_raw_coverage_e2e.rs +++ b/tests/memory_sync_providers_raw_coverage_e2e.rs @@ -25,6 +25,9 @@ use openhuman_core::openhuman::memory_sync::composio::bus::{ use openhuman_core::openhuman::memory_sync::composio::providers::clickup::ClickUpProvider; use openhuman_core::openhuman::memory_sync::composio::providers::github::GitHubProvider; use openhuman_core::openhuman::memory_sync::composio::providers::gmail::ingest as gmail_ingest; +use openhuman_core::openhuman::memory_sync::composio::providers::gmail::GmailProvider; +use openhuman_core::openhuman::memory_sync::composio::providers::linear::LinearProvider; +use openhuman_core::openhuman::memory_sync::composio::providers::notion::NotionProvider; use openhuman_core::openhuman::memory_sync::composio::providers::slack::ingest as slack_ingest; use openhuman_core::openhuman::memory_sync::composio::providers::slack::{ SlackMessage, SlackProvider, @@ -300,6 +303,8 @@ async fn configured_loopback_context( toolkit: toolkit.to_string(), connection_id: Some(connection_id.to_string()), usage: Default::default(), + max_items: None, + sync_depth_days: None, }; (config, ctx, server) } @@ -552,6 +557,8 @@ async fn github_clickup_and_composio_bus_cover_provider_branches() { toolkit: "clickup".to_string(), connection_id: Some("conn-clickup-round17".to_string()), usage: Default::default(), + max_items: None, + sync_depth_days: None, }; let clickup = ClickUpProvider::new(); let click_profile = clickup @@ -624,6 +631,572 @@ async fn github_clickup_and_composio_bus_cover_provider_branches() { server.abort(); } +// ───────────────────────────────────────────────────────────────────────── +// Slack cap enforcement +// +// Proves that max_items=N caps Slack ingestion to exactly N messages even +// when a single channel/page returns more than N messages. The mock returns +// 5 messages in one channel; with max_items=2 only 2 must be persisted. +// ───────────────────────────────────────────────────────────────────────── + +/// Build a loopback router for the Slack cap test. Returns 5 messages for +/// the single channel when `SLACK_FETCH_CONVERSATION_HISTORY` is called; +/// all other Slack bootstrap calls (auth, users, channel listing) return +/// minimal but valid responses. +fn slack_cap_router(requests: Arc>>) -> Router { + Router::new().route( + "/agent-integrations/composio/execute", + any(move |Json(body): Json| { + let requests = Arc::clone(&requests); + async move { + requests.lock().unwrap().push(body.clone()); + let tool = body.get("tool").and_then(Value::as_str).unwrap_or(""); + let resp = match tool { + "SLACK_TEST_AUTH" => execute_envelope(json!({ + "user_id": "UCAP", + "user": "capuser", + "team": "Cap Workspace", + "team_id": "TCAP", + "url": "https://cap.slack.com" + })), + "SLACK_RETRIEVE_DETAILED_USER_INFORMATION" => execute_envelope(json!({ + "user": { + "real_name": "Cap User", + "profile": { "email": "cap@example.test" } + } + })), + "SLACK_FETCH_TEAM_INFO" => execute_envelope(json!({ + "team": { "email_domain": "example.test" } + })), + // User directory — one member, no next page. + "SLACK_LIST_ALL_USERS" => execute_envelope(json!({ + "members": [ + { "id": "UCAP", "name": "capuser", "profile": { "real_name": "Cap User" } } + ], + "response_metadata": { "next_cursor": "" } + })), + // One channel, no next page. + "SLACK_LIST_CONVERSATIONS" => execute_envelope(json!({ + "channels": [ + { "id": "CCAP", "name": "cap-channel", "is_private": false } + ], + "response_metadata": { "next_cursor": "" } + })), + // Return 5 distinct messages for the single channel; + // the cap is 2 so only 2 must be persisted. + "SLACK_FETCH_CONVERSATION_HISTORY" => execute_envelope(json!({ + "messages": [ + { "ts": "1800000001.000001", "user": "UCAP", "text": "cap message 1", + "permalink": "https://cap.slack.com/archives/CCAP/p1800000001000001" }, + { "ts": "1800000002.000002", "user": "UCAP", "text": "cap message 2", + "permalink": "https://cap.slack.com/archives/CCAP/p1800000002000002" }, + { "ts": "1800000003.000003", "user": "UCAP", "text": "cap message 3", + "permalink": "https://cap.slack.com/archives/CCAP/p1800000003000003" }, + { "ts": "1800000004.000004", "user": "UCAP", "text": "cap message 4", + "permalink": "https://cap.slack.com/archives/CCAP/p1800000004000004" }, + { "ts": "1800000005.000005", "user": "UCAP", "text": "cap message 5", + "permalink": "https://cap.slack.com/archives/CCAP/p1800000005000005" } + ], + "response_metadata": { "next_cursor": "" } + })), + _ => execute_envelope(json!({})), + }; + Json(resp) + } + }), + ) +} + +#[tokio::test] +async fn slack_sync_max_items_caps_ingest_to_exact_count() { + let _guard = env_lock(); + let tmp = TempDir::new().expect("tempdir"); + let _workspace = EnvGuard::set_path("OPENHUMAN_WORKSPACE", tmp.path()); + let _home = EnvGuard::set_path("HOME", tmp.path()); + let _backend = EnvGuard::unset("BACKEND_URL"); + // Disable inter-call pacing so the test runs quickly. + let _pacing = EnvGuard::set("OPENHUMAN_SLACK_INTER_CALL_PACING_MS", "0"); + + let requests: Arc>> = Arc::new(Mutex::new(Vec::new())); + let (base, server) = loopback_router(slack_cap_router(Arc::clone(&requests))).await; + + let mut config = config_in(&tmp); + config.api_url = Some(base); + persist_config(&config).await; + store_session(&config); + memory_global::init(config.workspace_dir.clone()).expect("init global memory"); + + let ctx = ProviderContext { + config: Arc::new(config), + toolkit: "slack".to_string(), + connection_id: Some("conn-slack-cap".to_string()), + usage: Default::default(), + // The mock returns 5 messages; cap is 2 — only 2 must be ingested. + max_items: Some(2), + sync_depth_days: None, + }; + + let outcome = SlackProvider::new() + .sync(&ctx, SyncReason::ConnectionCreated) + .await + .expect("slack cap sync"); + + assert_eq!( + outcome.items_ingested, 2, + "max_items=2 must cap Slack ingest to exactly 2 even though the channel/page held 5" + ); + + server.abort(); +} + +// ───────────────────────────────────────────────────────────────────────── +// Gmail cap enforcement +// +// Proves that max_items=N caps Gmail ingestion to exactly N messages even +// when a single GMAIL_FETCH_EMAILS page returns more than N. The mock +// returns 5 messages in one page; with max_items=2 only 2 must be +// persisted. +// +// Gmail messages arrive in Composio's "upstream" shape before post-process +// reshapes them into the slim envelope: +// - messageId → id (used by MESSAGE_ID_PATHS dedup) +// - sender → from (used by ingest bucketing) +// - messageText → markdown body (from extract_markdown_body fallback) +// - messageTimestamp → date +// All 5 messages are unique (different messageId) and valid (non-empty +// messageText), so without the cap every one would be ingested. +// ───────────────────────────────────────────────────────────────────────── + +/// Build M distinct, valid Gmail messages in the upstream (pre-post-process) +/// Composio shape. Uses `messageId` / `sender` / `messageText` / +/// `messageTimestamp` so `reshape_message` maps them correctly into the slim +/// envelope that `ingest_page_into_memory_tree` expects. +fn gmail_cap_messages(m: usize) -> Vec { + (1..=m) + .map(|i| { + json!({ + "messageId": format!("gmail-cap-msg-{i}"), + "threadId": format!("thread-cap-{i}"), + "sender": format!("sender{i}@cap.example.test"), + "to": "recipient@cap.example.test", + "subject": format!("Cap test message {i}"), + "messageTimestamp": format!("2026-06-0{}T10:00:00Z", (i % 9) + 1), + "messageText": format!("Body of cap test message {i}. Sufficient content.") + }) + }) + .collect() +} + +/// Loopback router that answers the two Gmail sync tool calls. +/// GMAIL_GET_PROFILE returns a minimal valid profile; GMAIL_FETCH_EMAILS +/// returns one page containing all `messages` and no next-page token. +fn gmail_cap_router(messages: Vec, requests: Arc>>) -> Router { + Router::new().route( + "/agent-integrations/composio/execute", + any(move |Json(body): Json| { + let messages = messages.clone(); + let requests = Arc::clone(&requests); + async move { + requests.lock().unwrap().push(body.clone()); + let tool = body.get("tool").and_then(Value::as_str).unwrap_or(""); + let resp = match tool { + "GMAIL_GET_PROFILE" => execute_envelope(json!({ + "emailAddress": "cap@example.test", + "messagesTotal": messages.len() + })), + "GMAIL_FETCH_EMAILS" => execute_envelope(json!({ + "messages": messages, + "nextPageToken": "" + })), + _ => execute_envelope(json!({})), + }; + Json(resp) + } + }), + ) +} + +#[tokio::test] +async fn gmail_sync_max_items_caps_ingest_to_exact_count() { + let _guard = env_lock(); + let tmp = TempDir::new().expect("tempdir"); + let _workspace = EnvGuard::set_path("OPENHUMAN_WORKSPACE", tmp.path()); + let _home = EnvGuard::set_path("HOME", tmp.path()); + let _backend = EnvGuard::unset("BACKEND_URL"); + + // One page returns 5 valid messages; the cap is 2. + let requests: Arc>> = Arc::new(Mutex::new(Vec::new())); + let (base, server) = loopback_router(gmail_cap_router( + gmail_cap_messages(5), + Arc::clone(&requests), + )) + .await; + + let mut config = config_in(&tmp); + config.api_url = Some(base); + persist_config(&config).await; + store_session(&config); + memory_global::init(config.workspace_dir.clone()).expect("init global memory"); + + let ctx = ProviderContext { + config: Arc::new(config), + toolkit: "gmail".to_string(), + connection_id: Some("conn-gmail-cap".to_string()), + usage: Default::default(), + // Mock returns 5 messages; cap is 2 — only 2 must be ingested. + max_items: Some(2), + sync_depth_days: None, + }; + + let outcome = GmailProvider::new() + .sync(&ctx, SyncReason::ConnectionCreated) + .await + .expect("gmail cap sync"); + + assert_eq!( + outcome.items_ingested, 2, + "max_items=2 must cap Gmail ingest to exactly 2 even though the page held 5" + ); + + server.abort(); +} + +// ───────────────────────────────────────────────────────────────────────── +// Notion cap enforcement +// +// Proves that max_items=N caps Notion ingestion to exactly N pages even +// when a single NOTION_FETCH_DATA page returns more than N. The mock +// returns 5 pages; with max_items=2 only 2 must be persisted. +// +// sync_depth_days is None and all items carry a recent last_edited_time +// so no depth-filter skipping happens — the cap is the only thing that +// limits ingestion. +// ───────────────────────────────────────────────────────────────────────── + +/// Build M distinct, valid Notion page objects. Each has a unique id and a +/// recent `last_edited_time` so the depth filter (when enabled) would keep +/// them all. +fn notion_cap_pages(m: usize) -> Vec { + (1..=m) + .map(|i| { + json!({ + "id": format!("notion-cap-page-{i:04}"), + "object": "page", + "last_edited_time": format!("2026-06-0{}T10:00:00.000Z", (i % 9) + 1), + "properties": { + "Name": { + "type": "title", + "title": [{ "plain_text": format!("Cap page {i}") }] + } + }, + "url": format!("https://www.notion.so/cap-page-{i}") + }) + }) + .collect() +} + +/// Loopback router for the Notion cap test. +/// NOTION_GET_ABOUT_ME returns a minimal identity; NOTION_FETCH_DATA returns +/// one page with all `pages` as results and no next_cursor. +fn notion_cap_router(pages: Vec, requests: Arc>>) -> Router { + Router::new().route( + "/agent-integrations/composio/execute", + any(move |Json(body): Json| { + let pages = pages.clone(); + let requests = Arc::clone(&requests); + async move { + requests.lock().unwrap().push(body.clone()); + let tool = body.get("tool").and_then(Value::as_str).unwrap_or(""); + let resp = match tool { + "NOTION_GET_ABOUT_ME" => execute_envelope(json!({ + "id": "notion-cap-user", + "name": "Cap User", + "type": "bot" + })), + "NOTION_FETCH_DATA" => execute_envelope(json!({ + "results": pages, + "next_cursor": null, + "has_more": false + })), + _ => execute_envelope(json!({})), + }; + Json(resp) + } + }), + ) +} + +#[tokio::test] +async fn notion_sync_max_items_caps_ingest_to_exact_count() { + let _guard = env_lock(); + let tmp = TempDir::new().expect("tempdir"); + let _workspace = EnvGuard::set_path("OPENHUMAN_WORKSPACE", tmp.path()); + let _home = EnvGuard::set_path("HOME", tmp.path()); + let _backend = EnvGuard::unset("BACKEND_URL"); + + // One page returns 5 valid Notion pages; the cap is 2. + let requests: Arc>> = Arc::new(Mutex::new(Vec::new())); + let (base, server) = loopback_router(notion_cap_router( + notion_cap_pages(5), + Arc::clone(&requests), + )) + .await; + + let mut config = config_in(&tmp); + config.api_url = Some(base); + persist_config(&config).await; + store_session(&config); + memory_global::init(config.workspace_dir.clone()).expect("init global memory"); + + let ctx = ProviderContext { + config: Arc::new(config), + toolkit: "notion".to_string(), + connection_id: Some("conn-notion-cap".to_string()), + usage: Default::default(), + // Mock returns 5 pages; cap is 2 — only 2 must be ingested. + max_items: Some(2), + sync_depth_days: None, + }; + + let outcome = NotionProvider::new() + .sync(&ctx, SyncReason::ConnectionCreated) + .await + .expect("notion cap sync"); + + assert_eq!( + outcome.items_ingested, 2, + "max_items=2 must cap Notion ingest to exactly 2 even though the page held 5" + ); + + server.abort(); +} + +// ───────────────────────────────────────────────────────────────────────── +// Linear cap enforcement +// +// Proves that max_items=N caps Linear ingestion to exactly N issues even +// when a single LINEAR_LIST_LINEAR_ISSUES page returns more than N. +// +// Tool sequence: +// 1. LINEAR_LIST_LINEAR_USERS { isMe: true } → viewer id +// 2. LINEAR_LIST_LINEAR_ISSUES (assigneeId=...) → nodes of issues +// +// Issues are in Linear's `{ nodes: [...], pageInfo: {...} }` shape. +// We return hasNextPage=false so pagination stops after one page. +// Each issue has a unique id and a recent updatedAt. +// ───────────────────────────────────────────────────────────────────────── + +/// Build M distinct, valid Linear issue objects. +fn linear_cap_issues(m: usize) -> Vec { + (1..=m) + .map(|i| { + json!({ + "id": format!("linear-cap-issue-{i:04}"), + "identifier": format!("ENG-{i}"), + "title": format!("Cap issue {i}"), + "updatedAt": format!("2026-06-0{}T10:00:00.000Z", (i % 9) + 1), + "url": format!("https://linear.app/cap/issue/ENG-{i}"), + "description": format!("Description for cap issue {i}.") + }) + }) + .collect() +} + +/// Loopback router for the Linear cap test. +/// LINEAR_LIST_LINEAR_USERS returns a single viewer node so `resolve_viewer_id` +/// succeeds. LINEAR_LIST_LINEAR_ISSUES returns one page with all issues and no +/// next-page cursor (hasNextPage=false). +fn linear_cap_router(issues: Vec, requests: Arc>>) -> Router { + Router::new().route( + "/agent-integrations/composio/execute", + any(move |Json(body): Json| { + let issues = issues.clone(); + let requests = Arc::clone(&requests); + async move { + requests.lock().unwrap().push(body.clone()); + let tool = body.get("tool").and_then(Value::as_str).unwrap_or(""); + let resp = match tool { + "LINEAR_LIST_LINEAR_USERS" => execute_envelope(json!({ + "nodes": [ + { "id": "linear-cap-viewer", "name": "Cap Viewer", "email": "cap@linear.test" } + ] + })), + "LINEAR_LIST_LINEAR_ISSUES" => execute_envelope(json!({ + "nodes": issues, + "pageInfo": { + "hasNextPage": false, + "endCursor": null + } + })), + _ => execute_envelope(json!({})), + }; + Json(resp) + } + }), + ) +} + +#[tokio::test] +async fn linear_sync_max_items_caps_ingest_to_exact_count() { + let _guard = env_lock(); + let tmp = TempDir::new().expect("tempdir"); + let _workspace = EnvGuard::set_path("OPENHUMAN_WORKSPACE", tmp.path()); + let _home = EnvGuard::set_path("HOME", tmp.path()); + let _backend = EnvGuard::unset("BACKEND_URL"); + + // One page returns 5 valid Linear issues; the cap is 2. + let requests: Arc>> = Arc::new(Mutex::new(Vec::new())); + let (base, server) = loopback_router(linear_cap_router( + linear_cap_issues(5), + Arc::clone(&requests), + )) + .await; + + let mut config = config_in(&tmp); + config.api_url = Some(base); + persist_config(&config).await; + store_session(&config); + memory_global::init(config.workspace_dir.clone()).expect("init global memory"); + + let ctx = ProviderContext { + config: Arc::new(config), + toolkit: "linear".to_string(), + connection_id: Some("conn-linear-cap".to_string()), + usage: Default::default(), + // Mock returns 5 issues; cap is 2 — only 2 must be ingested. + max_items: Some(2), + sync_depth_days: None, + }; + + let outcome = LinearProvider::new() + .sync(&ctx, SyncReason::ConnectionCreated) + .await + .expect("linear cap sync"); + + assert_eq!( + outcome.items_ingested, 2, + "max_items=2 must cap Linear ingest to exactly 2 even though the page held 5" + ); + + server.abort(); +} + +// ───────────────────────────────────────────────────────────────────────── +// ClickUp cap enforcement +// +// Proves that max_items=N caps ClickUp ingestion to exactly N tasks even +// when a single CLICKUP_GET_FILTERED_TEAM_TASKS page returns more than N. +// +// Tool sequence: +// 1. CLICKUP_GET_AUTHORIZED_USER → user numeric id +// 2. CLICKUP_GET_AUTHORIZED_TEAMS_WORKSPACES → one workspace +// 3. CLICKUP_GET_FILTERED_TEAM_TASKS → tasks page +// +// The tasks page returns 5 items; with max_items=2 only 2 must be +// persisted. Because INITIAL_PAGE_SIZE=100 for ConnectionCreated and we +// return only 5 tasks (< 100), the short-page guard stops the loop so no +// second page is requested, matching the cap path cleanly. +// ───────────────────────────────────────────────────────────────────────── + +/// Build M distinct, valid ClickUp task objects. +fn clickup_cap_tasks(m: usize) -> Vec { + // date_updated must be large enough to sort lexicographically correctly + // but recent enough not to trip any depth filter. We use ms-since-epoch + // strings in the vicinity of mid-2026 (≈1780000000000 ms). + (1..=m) + .map(|i| { + json!({ + "id": format!("clickup-cap-task-{i:04}"), + "name": format!("Cap task {i}"), + "text_content": format!("Content for cap task {i}."), + "status": { "status": "open" }, + "date_updated": format!("{}", 1_780_000_000_000_u64 + i as u64 * 1000), + "url": format!("https://app.clickup.com/t/clickup-cap-task-{i:04}") + }) + }) + .collect() +} + +/// Loopback router for the ClickUp cap test. +fn clickup_cap_router(tasks: Vec, requests: Arc>>) -> Router { + Router::new().route( + "/agent-integrations/composio/execute", + any(move |Json(body): Json| { + let tasks = tasks.clone(); + let requests = Arc::clone(&requests); + async move { + requests.lock().unwrap().push(body.clone()); + let tool = body.get("tool").and_then(Value::as_str).unwrap_or(""); + let resp = match tool { + "CLICKUP_GET_AUTHORIZED_USER" => execute_envelope(json!({ + "user": { + "id": 42, + "username": "cap-user", + "email": "cap@clickup.test", + "profilePicture": null + } + })), + "CLICKUP_GET_AUTHORIZED_TEAMS_WORKSPACES" => execute_envelope(json!({ + "teams": [ + { "id": "ws-cap-01", "name": "Cap Workspace" } + ] + })), + "CLICKUP_GET_FILTERED_TEAM_TASKS" => execute_envelope(json!({ + "tasks": tasks + })), + _ => execute_envelope(json!({})), + }; + Json(resp) + } + }), + ) +} + +#[tokio::test] +async fn clickup_sync_max_items_caps_ingest_to_exact_count() { + let _guard = env_lock(); + let tmp = TempDir::new().expect("tempdir"); + let _workspace = EnvGuard::set_path("OPENHUMAN_WORKSPACE", tmp.path()); + let _home = EnvGuard::set_path("HOME", tmp.path()); + let _backend = EnvGuard::unset("BACKEND_URL"); + + // One workspace, one page returning 5 valid tasks; the cap is 2. + let requests: Arc>> = Arc::new(Mutex::new(Vec::new())); + let (base, server) = loopback_router(clickup_cap_router( + clickup_cap_tasks(5), + Arc::clone(&requests), + )) + .await; + + let mut config = config_in(&tmp); + config.api_url = Some(base); + persist_config(&config).await; + store_session(&config); + memory_global::init(config.workspace_dir.clone()).expect("init global memory"); + + let ctx = ProviderContext { + config: Arc::new(config), + toolkit: "clickup".to_string(), + connection_id: Some("conn-clickup-cap".to_string()), + usage: Default::default(), + // Mock returns 5 tasks; cap is 2 — only 2 must be ingested. + max_items: Some(2), + sync_depth_days: None, + }; + + let outcome = ClickUpProvider::new() + .sync(&ctx, SyncReason::ConnectionCreated) + .await + .expect("clickup cap sync"); + + assert_eq!( + outcome.items_ingested, 2, + "max_items=2 must cap ClickUp ingest to exactly 2 even though the page held 5" + ); + + server.abort(); +} + fn walk_files(root: &Path) -> Vec { let mut out = Vec::new(); if !root.exists() { @@ -646,3 +1219,185 @@ fn walk_files(root: &Path) -> Vec { } out } + +// ───────────────────────────────────────────────────────────────────────── +// Sync-cap enforcement: max_items and sync_depth_days +// +// These verify the user-facing promise of the per-source sync settings: +// - max_items=N ingests AT MOST N items, even when a single API page +// returns more (precise mid-page cap, not just a page cap). +// - sync_depth_days injects an `updated:>{floor}` date filter into the +// GitHub search query so only recent items are requested. +// The cap logic is shared verbatim across the gmail/notion/linear/clickup +// providers, so GitHub stands in for all of them here. +// ───────────────────────────────────────────────────────────────────────── + +/// Build `n` distinct, valid GitHub search items (each with an id + updated_at). +fn github_issue_items(n: usize) -> Vec { + (1..=n) + .map(|i| { + json!({ + "id": 3000 + i, + "title": format!("Cap issue {i}"), + "body": "cap enforcement body", + "state": "open", + "updated_at": format!("2026-05-{:02}T10:00:00Z", 10 + i), + "html_url": format!("https://github.com/tinyhumansai/openhuman/issues/{i}") + }) + }) + .collect() +} + +/// Loopback router that answers the two GitHub sync tools. The search tool +/// always returns `items` (one page), and every request body is captured. +fn github_cap_router(items: Vec, requests: Arc>>) -> Router { + Router::new().route( + "/agent-integrations/composio/execute", + any(move |Json(body): Json| { + let items = items.clone(); + let requests = Arc::clone(&requests); + async move { + requests.lock().unwrap().push(body.clone()); + let tool = body.get("tool").and_then(Value::as_str).unwrap_or(""); + let resp = match tool { + "GITHUB_GET_THE_AUTHENTICATED_USER" => execute_envelope(json!({ + "login": "octo-cap", + "html_url": "https://github.com/octo-cap" + })), + "GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS" => execute_envelope(json!({ + "items": items, + "total_count": items.len() + })), + _ => execute_envelope(json!({})), + }; + Json(resp) + } + }), + ) +} + +async fn github_cap_context( + tmp: &TempDir, + items: Vec, + requests: Arc>>, +) -> (Config, tokio::task::JoinHandle<()>) { + let (base, server) = loopback_router(github_cap_router(items, requests)).await; + let mut config = config_in(tmp); + config.api_url = Some(base); + persist_config(&config).await; + store_session(&config); + memory_global::init(config.workspace_dir.clone()).expect("init global memory"); + (config, server) +} + +fn github_ctx( + config: &Config, + max_items: Option, + sync_depth_days: Option, +) -> ProviderContext { + ProviderContext { + config: Arc::new(config.clone()), + toolkit: "github".to_string(), + connection_id: Some("conn-cap".to_string()), + usage: Default::default(), + max_items, + sync_depth_days, + } +} + +#[tokio::test] +async fn github_sync_max_items_caps_ingest_to_exact_count() { + let _guard = env_lock(); + let tmp = TempDir::new().expect("tempdir"); + let _workspace = EnvGuard::set_path("OPENHUMAN_WORKSPACE", tmp.path()); + let _home = EnvGuard::set_path("HOME", tmp.path()); + let _backend = EnvGuard::unset("BACKEND_URL"); + + // One page returns 5 valid items; the cap is 2. + let requests = Arc::new(Mutex::new(Vec::new())); + let (config, server) = + github_cap_context(&tmp, github_issue_items(5), Arc::clone(&requests)).await; + + let outcome = GitHubProvider::new() + .sync( + &github_ctx(&config, Some(2), None), + SyncReason::ConnectionCreated, + ) + .await + .expect("github sync"); + + assert_eq!( + outcome.items_ingested, 2, + "max_items=2 must cap ingest to exactly 2 even though the page held 5" + ); + server.abort(); +} + +#[tokio::test] +async fn github_sync_without_max_items_ingests_full_page() { + let _guard = env_lock(); + let tmp = TempDir::new().expect("tempdir"); + let _workspace = EnvGuard::set_path("OPENHUMAN_WORKSPACE", tmp.path()); + let _home = EnvGuard::set_path("HOME", tmp.path()); + let _backend = EnvGuard::unset("BACKEND_URL"); + + // Control: no cap → every valid item on the page is ingested. + let requests = Arc::new(Mutex::new(Vec::new())); + let (config, server) = + github_cap_context(&tmp, github_issue_items(5), Arc::clone(&requests)).await; + + let outcome = GitHubProvider::new() + .sync( + &github_ctx(&config, None, None), + SyncReason::ConnectionCreated, + ) + .await + .expect("github sync"); + + assert_eq!( + outcome.items_ingested, 5, + "with no max_items cap, all 5 page items must be ingested" + ); + server.abort(); +} + +#[tokio::test] +async fn github_sync_depth_days_injects_updated_floor_into_query() { + let _guard = env_lock(); + let tmp = TempDir::new().expect("tempdir"); + let _workspace = EnvGuard::set_path("OPENHUMAN_WORKSPACE", tmp.path()); + let _home = EnvGuard::set_path("HOME", tmp.path()); + let _backend = EnvGuard::unset("BACKEND_URL"); + + let requests = Arc::new(Mutex::new(Vec::new())); + let (config, server) = + github_cap_context(&tmp, github_issue_items(1), Arc::clone(&requests)).await; + + GitHubProvider::new() + .sync( + &github_ctx(&config, None, Some(7)), + SyncReason::ConnectionCreated, + ) + .await + .expect("github sync"); + + // The search request must carry an `updated:>{date}` floor derived from the + // 7-day window, proving sync_depth_days actually narrows what is fetched. + let reqs = requests.lock().unwrap(); + let search = reqs + .iter() + .find(|b| { + b.get("tool").and_then(Value::as_str) == Some("GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS") + }) + .expect("a search request was issued"); + let q = search + .get("arguments") + .and_then(|a| a.get("q")) + .and_then(Value::as_str) + .unwrap_or(""); + assert!( + q.contains("updated:>"), + "sync_depth_days=7 must inject an `updated:>` date floor, got query: {q}" + ); + server.abort(); +} diff --git a/tests/memory_sync_round23_raw_coverage_e2e.rs b/tests/memory_sync_round23_raw_coverage_e2e.rs index 9ebbf36ec..956782c9e 100644 --- a/tests/memory_sync_round23_raw_coverage_e2e.rs +++ b/tests/memory_sync_round23_raw_coverage_e2e.rs @@ -165,6 +165,8 @@ async fn configured_context( toolkit: toolkit.to_string(), connection_id: Some(connection_id.to_string()), usage: Default::default(), + max_items: None, + sync_depth_days: None, }; (config, ctx, server) } diff --git a/tests/memory_sync_slack_bus_raw_coverage_e2e.rs b/tests/memory_sync_slack_bus_raw_coverage_e2e.rs index e3726de34..c153ac2cc 100644 --- a/tests/memory_sync_slack_bus_raw_coverage_e2e.rs +++ b/tests/memory_sync_slack_bus_raw_coverage_e2e.rs @@ -260,6 +260,8 @@ async fn configured_loopback_context( toolkit: "slack".to_string(), connection_id: Some("conn-slack-round19".to_string()), usage: Default::default(), + max_items: None, + sync_depth_days: None, }; (config, ctx, server) } diff --git a/tests/memory_sync_sources_raw_coverage_e2e.rs b/tests/memory_sync_sources_raw_coverage_e2e.rs index 9c0702457..65f8ed3bc 100644 --- a/tests/memory_sync_sources_raw_coverage_e2e.rs +++ b/tests/memory_sync_sources_raw_coverage_e2e.rs @@ -508,6 +508,8 @@ async fn composio_providers_fetch_profiles_tasks_and_cover_error_branches() { toolkit: "github".to_string(), connection_id: Some("conn-github".to_string()), usage: Default::default(), + max_items: None, + sync_depth_days: None, }; let github = GitHubProvider::new(); let github_profile = github diff --git a/tests/memory_sync_tree_round21_raw_coverage_e2e.rs b/tests/memory_sync_tree_round21_raw_coverage_e2e.rs index 7343e8602..f25cfe9c4 100644 --- a/tests/memory_sync_tree_round21_raw_coverage_e2e.rs +++ b/tests/memory_sync_tree_round21_raw_coverage_e2e.rs @@ -239,6 +239,8 @@ async fn configured_loopback_context( toolkit: toolkit.to_string(), connection_id: Some(connection_id.to_string()), usage: Default::default(), + max_items: None, + sync_depth_days: None, }; (config, ctx, server) } diff --git a/tests/memory_threads_raw_coverage_e2e.rs b/tests/memory_threads_raw_coverage_e2e.rs index 406fad5c9..d7081b92f 100644 --- a/tests/memory_threads_raw_coverage_e2e.rs +++ b/tests/memory_threads_raw_coverage_e2e.rs @@ -3047,6 +3047,8 @@ async fn memory_sync_provider_trait_defaults_and_connection_hook_are_determinist toolkit: "raw_coverage".into(), connection_id: Some("conn-1".into()), usage: Default::default(), + max_items: None, + sync_depth_days: None, }; let provider = RawCoverageProvider { fail_profile: true }; assert_eq!(provider.sync_interval_secs(), Some(15 * 60)); diff --git a/tests/memory_tree_sync_raw_coverage_e2e.rs b/tests/memory_tree_sync_raw_coverage_e2e.rs index bd31d8666..821e75fae 100644 --- a/tests/memory_tree_sync_raw_coverage_e2e.rs +++ b/tests/memory_tree_sync_raw_coverage_e2e.rs @@ -599,6 +599,8 @@ async fn default_composio_provider_hooks_return_expected_noop_shapes() { toolkit: "round14".into(), connection_id: Some("conn-round14".into()), usage: Default::default(), + max_items: None, + sync_depth_days: None, }; let provider = MinimalProvider; assert_eq!(provider.sync_interval_secs(), None);