mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-27 21:08:00 +00:00
feat(memory-sources): default sync sources on with conservative caps, per-source settings & All In (#3304)
This commit is contained in:
@@ -300,3 +300,15 @@ Quick reference for anyone starting with Claude on this project. Updated by the
|
||||
|
||||
- **Upstream `main` has 5 Vitest failures and 4 TypeScript compile errors** — Caused by missing iOS experimental dependencies: `@noble/ciphers/chacha`, `@noble/ciphers/webcrypto`, `qrcode.react`, `@tauri-apps/plugin-barcode-scanner`. Breaks `pnpm compile`, `pnpm build`, `pnpm test:coverage` on a clean checkout. Always verify by stashing changes and running checks on the base branch before blaming your PR.
|
||||
- **`cargo fmt` must run after codecrusher** — codecrusher does not reliably produce `cargo fmt`-clean Rust. Always run `cargo fmt --manifest-path Cargo.toml` after codecrusher finishes and before committing.
|
||||
|
||||
## Memory Sync Sources — Defaults ON + Per-Source UI (Issue #3293)
|
||||
|
||||
- **Conservative caps registry** — `composio_defaults_for_toolkit(toolkit) -> (max_items, sync_depth_days)` in `src/openhuman/memory_sources/registry.rs` is the single source of truth. Values: gmail 100/30, slack 50/14, notion 30/30, linear 50/30, clickup 50/30, github 50/30, generic 30/14. Non-Composio defaults (GithubRepo 10PR/10issue/50commit, RSS 20, Twitter 7d) live in `apply_kind_defaults` in `rpc.rs`.
|
||||
- **`upsert_composio_source` now defaults ON** — Registers `enabled: true` with caps applied from the registry. Previously registered `enabled: false` with no caps.
|
||||
- **Cap enforcement: 3 construction sites, not 1** — `ProviderContext` (`memory_sync/composio/providers/types.rs`) carries `max_items`/`sync_depth_days`. All three sites must populate caps from the registry entry: `composio/mod.rs run_connection_sync`, `composio/periodic.rs`, `composio/bus.rs`. Each provider reads `ctx.max_items`/`ctx.sync_depth_days` for pagination + date clamping. Shared helpers: `pages_for_max_items` / `epoch_floor_from_depth` in `providers/helpers.rs`.
|
||||
- **First-sync gotcha on new connections** — `bus.rs on_connection_created` fires BEFORE `upsert_composio_source`, so caps are (None, None) on the brand-new connection's first sync — bounded only by internal `MAX_PAGES`, not registry caps. Documented in code; intentional.
|
||||
- **`memory_sources_apply_all_in` RPC** — Zero params → `{ sources, sync_triggered }`. Enables all sources, clears caps to None (falls back to internal `MAX_PAGES` ceilings, ~500 items, not truly unlimited), triggers sync per source.
|
||||
- **Retroactive migration** — `apply_composio_source_caps_migration` in `reconcile.rs`, guarded by `Config.composio_source_caps_migrated: bool` (`#[serde(default)]`), runs once from `ensure_composio_sources`. Only touches Composio entries with `!enabled && max_items.is_none() && sync_depth_days.is_none()` — never overwrites user-customized caps.
|
||||
- **`MemorySourcePatch` was missing limit fields** — `max_commits`, `max_issues`, `max_prs` were absent from `MemorySourcePatch`, `update_source`, and the update schema. Also missing from the TS `MemorySourceEntry` interface in `memorySourcesService.ts`. Both were fixed in this issue.
|
||||
- **Per-source settings UI** — `SourceSettingsPanel.tsx` (sibling of `MemorySourcesRegistry.tsx`) with a `KIND_FIELDS` map driving which limit fields appear per source kind. Empty input = omit from patch (use default); number = set. "All In" button uses `ConfirmationModal` (primary-500 prominent). Toasts via existing `onToast` prop.
|
||||
- **`pnpm i18n:english:check` is pre-failing** — Exits with `total unexpected English: 1312` on a clean base tree. Confirm your new keys aren't among the failures rather than expecting exit 0. `pnpm i18n:check` (key parity) is the real gate.
|
||||
|
||||
@@ -8,10 +8,11 @@
|
||||
* row dispatches `openhuman.memory_sources_sync` which runs in the
|
||||
* background and emits MemorySyncStageChanged events.
|
||||
*/
|
||||
import { useCallback, useEffect, useMemo, useState } from 'react';
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
||||
|
||||
import { useT } from '../../lib/i18n/I18nContext';
|
||||
import {
|
||||
applyAllIn,
|
||||
type FreshnessLabel,
|
||||
listMemorySources,
|
||||
type MemorySourceEntry,
|
||||
@@ -23,9 +24,14 @@ import {
|
||||
syncMemorySource,
|
||||
updateMemorySource,
|
||||
} from '../../services/memorySourcesService';
|
||||
import type { ToastNotification } from '../../types/intelligence';
|
||||
import type {
|
||||
ConfirmationModal as ConfirmationModalType,
|
||||
ToastNotification,
|
||||
} from '../../types/intelligence';
|
||||
import { memoryTreeFlushSource } from '../../utils/tauriCommands/memoryTree';
|
||||
import { AddMemorySourceDialog } from './AddMemorySourceDialog';
|
||||
import { ConfirmationModal } from './ConfirmationModal';
|
||||
import { SourceSettingsPanel } from './SourceSettingsPanel';
|
||||
|
||||
interface MemorySourcesRegistryProps {
|
||||
onToast?: (toast: Omit<ToastNotification, 'id'>) => void;
|
||||
@@ -59,6 +65,10 @@ export function MemorySourcesRegistry({
|
||||
const [syncingId, setSyncingId] = useState<string | null>(null);
|
||||
const [buildingId, setBuildingId] = useState<string | null>(null);
|
||||
const [syncProgress, setSyncProgress] = useState<Map<string, SyncProgress>>(new Map());
|
||||
const [allInModalOpen, setAllInModalOpen] = useState(false);
|
||||
const [applyingAllIn, setApplyingAllIn] = useState(false);
|
||||
const allInInFlightRef = useRef(false);
|
||||
const [expandedSettingsId, setExpandedSettingsId] = useState<string | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
const handler = (e: Event) => {
|
||||
@@ -223,23 +233,84 @@ export function MemorySourcesRegistry({
|
||||
[onToast, refresh, t]
|
||||
);
|
||||
|
||||
const handleConfirmAllIn = useCallback(async () => {
|
||||
if (allInInFlightRef.current) return;
|
||||
allInInFlightRef.current = true;
|
||||
setApplyingAllIn(true);
|
||||
try {
|
||||
const result = await applyAllIn();
|
||||
setSources(result.sources);
|
||||
onToast?.({ type: 'success', title: t('memorySources.allIn.success') });
|
||||
} catch (err) {
|
||||
onToast?.({
|
||||
type: 'error',
|
||||
title: t('memorySources.allIn.failed'),
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
} finally {
|
||||
allInInFlightRef.current = false;
|
||||
setApplyingAllIn(false);
|
||||
setAllInModalOpen(false);
|
||||
}
|
||||
}, [onToast, t]);
|
||||
|
||||
const handleSettingsSaved = useCallback((updated: MemorySourceEntry) => {
|
||||
setSources(prev => prev.map(s => (s.id === updated.id ? updated : s)));
|
||||
}, []);
|
||||
|
||||
const handleToggleSettings = useCallback((sourceId: string) => {
|
||||
setExpandedSettingsId(prev => (prev === sourceId ? null : sourceId));
|
||||
}, []);
|
||||
|
||||
const allInModal: ConfirmationModalType = {
|
||||
isOpen: allInModalOpen,
|
||||
title: t('memorySources.allIn.title'),
|
||||
message: t('memorySources.allIn.message'),
|
||||
confirmText: t('memorySources.allIn.confirm'),
|
||||
cancelText: t('memorySources.allIn.cancel'),
|
||||
destructive: false,
|
||||
onConfirm: () => {
|
||||
void handleConfirmAllIn();
|
||||
},
|
||||
onCancel: () => {
|
||||
setAllInModalOpen(false);
|
||||
},
|
||||
};
|
||||
|
||||
return (
|
||||
<section
|
||||
className="rounded-lg border border-stone-200 bg-white p-4 dark:border-neutral-800 dark:bg-neutral-900"
|
||||
data-testid="memory-sources">
|
||||
<header className="mb-3 flex items-center justify-between">
|
||||
<header className="mb-3 flex items-center justify-between gap-2">
|
||||
<h3 className="text-sm font-semibold text-stone-700 dark:text-neutral-200">
|
||||
{t('memorySources.title')}
|
||||
</h3>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setDialogOpen(true)}
|
||||
className="inline-flex items-center gap-1 rounded-md bg-primary-500 px-3 py-1.5
|
||||
text-xs font-semibold text-white shadow-sm transition-colors
|
||||
hover:bg-primary-600 focus:outline-none focus:ring-2 focus:ring-primary-200">
|
||||
<PlusIcon />
|
||||
{t('memorySources.addSource')}
|
||||
</button>
|
||||
<div className="flex items-center gap-2">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setAllInModalOpen(true)}
|
||||
disabled={applyingAllIn}
|
||||
data-testid="all-in-button"
|
||||
className="inline-flex items-center gap-1 rounded-md border border-primary-300
|
||||
bg-white px-3 py-1.5 text-xs font-semibold text-primary-600
|
||||
shadow-sm transition-colors hover:bg-primary-50
|
||||
disabled:cursor-not-allowed disabled:opacity-50
|
||||
dark:border-primary-500/30 dark:bg-neutral-900 dark:text-primary-400
|
||||
dark:hover:bg-primary-500/10
|
||||
focus:outline-none focus:ring-2 focus:ring-primary-200">
|
||||
<AllInIcon />
|
||||
{t('memorySources.allIn.button')}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setDialogOpen(true)}
|
||||
className="inline-flex items-center gap-1 rounded-md bg-primary-500 px-3 py-1.5
|
||||
text-xs font-semibold text-white shadow-sm transition-colors
|
||||
hover:bg-primary-600 focus:outline-none focus:ring-2 focus:ring-primary-200">
|
||||
<PlusIcon />
|
||||
{t('memorySources.addSource')}
|
||||
</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
{loading ? (
|
||||
@@ -256,10 +327,14 @@ export function MemorySourcesRegistry({
|
||||
isSyncing={syncingId === source.id}
|
||||
isBuilding={buildingId === source.id}
|
||||
progress={syncProgress.get(source.id) ?? null}
|
||||
settingsExpanded={expandedSettingsId === source.id}
|
||||
onToggle={handleToggle}
|
||||
onRemove={handleRemove}
|
||||
onSync={handleSync}
|
||||
onBuild={handleBuild}
|
||||
onToggleSettings={handleToggleSettings}
|
||||
onSettingsSaved={handleSettingsSaved}
|
||||
onToast={onToast}
|
||||
/>
|
||||
))}
|
||||
</ul>
|
||||
@@ -270,6 +345,10 @@ export function MemorySourcesRegistry({
|
||||
onClose={() => setDialogOpen(false)}
|
||||
onAdded={handleAdded}
|
||||
/>
|
||||
|
||||
{allInModalOpen && (
|
||||
<ConfirmationModal modal={allInModal} onClose={() => setAllInModalOpen(false)} />
|
||||
)}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
@@ -280,10 +359,14 @@ interface SourceRowProps {
|
||||
isSyncing: boolean;
|
||||
isBuilding: boolean;
|
||||
progress: SyncProgress | null;
|
||||
settingsExpanded: boolean;
|
||||
onToggle: (source: MemorySourceEntry) => void;
|
||||
onRemove: (source: MemorySourceEntry) => void;
|
||||
onSync: (source: MemorySourceEntry) => void;
|
||||
onBuild: (source: MemorySourceEntry) => void;
|
||||
onToggleSettings: (sourceId: string) => void;
|
||||
onSettingsSaved: (updated: MemorySourceEntry) => void;
|
||||
onToast?: (toast: Omit<ToastNotification, 'id'>) => void;
|
||||
}
|
||||
|
||||
function SourceRow({
|
||||
@@ -292,10 +375,14 @@ function SourceRow({
|
||||
isSyncing,
|
||||
isBuilding,
|
||||
progress,
|
||||
settingsExpanded,
|
||||
onToggle,
|
||||
onRemove,
|
||||
onSync,
|
||||
onBuild,
|
||||
onToggleSettings,
|
||||
onSettingsSaved,
|
||||
onToast,
|
||||
}: SourceRowProps) {
|
||||
const { t } = useT();
|
||||
const icon = SOURCE_KIND_ICONS[source.kind] ?? '📄';
|
||||
@@ -304,125 +391,146 @@ function SourceRow({
|
||||
const lastSync = status ? relativeTimestamp(status.last_chunk_at_ms, t) : null;
|
||||
|
||||
return (
|
||||
<li
|
||||
className="flex flex-col gap-2 py-3 sm:flex-row sm:items-start sm:justify-between"
|
||||
data-testid={`memory-source-row-${source.kind}`}>
|
||||
<div className="min-w-0 flex-1">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<span className="text-base">{icon}</span>
|
||||
<span
|
||||
className={`truncate text-sm font-medium ${
|
||||
source.enabled
|
||||
? 'text-stone-900 dark:text-neutral-100'
|
||||
: 'text-stone-400 line-through dark:text-neutral-500'
|
||||
}`}>
|
||||
{source.label}
|
||||
</span>
|
||||
<span className="rounded-md bg-stone-100 px-1.5 py-0.5 text-[10px] font-medium text-stone-500 dark:bg-neutral-800 dark:text-neutral-400">
|
||||
{kindLabel}
|
||||
</span>
|
||||
{status && status.chunks_synced > 0 && <FreshnessPill freshness={status.freshness} />}
|
||||
</div>
|
||||
{detail && (
|
||||
<p className="mt-0.5 truncate pl-7 text-xs text-stone-400 dark:text-neutral-500">
|
||||
{detail}
|
||||
</p>
|
||||
)}
|
||||
{progress && (
|
||||
<div className="mt-2 pl-7">
|
||||
<div className="flex items-center gap-2 text-xs text-stone-500 dark:text-neutral-400">
|
||||
<span className="capitalize">{progress.stage}</span>
|
||||
{progress.percent !== null && (
|
||||
<span className="font-medium text-primary-600 dark:text-primary-400">
|
||||
{progress.percent}%
|
||||
</span>
|
||||
)}
|
||||
{progress.detail && (
|
||||
<span className="truncate text-stone-400 dark:text-neutral-500">
|
||||
{progress.detail}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<div className="mt-1 h-1.5 w-full overflow-hidden rounded-full bg-stone-200 dark:bg-neutral-700">
|
||||
<div
|
||||
className="h-full rounded-full bg-primary-500 transition-all duration-300"
|
||||
style={{
|
||||
width: `${progress.percent ?? (progress.stage === 'fetching' ? 10 : 5)}%`,
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{!progress && status && (status.chunks_synced > 0 || status.chunks_pending > 0) && (
|
||||
<div className="mt-1 flex flex-wrap items-center gap-x-3 gap-y-0.5 pl-7 text-xs text-stone-500 dark:text-neutral-400">
|
||||
<span>
|
||||
{status.chunks_synced.toLocaleString()} {t('sync.chunks')}
|
||||
<li className="flex flex-col gap-2 py-3" data-testid={`memory-source-row-${source.kind}`}>
|
||||
<div className="flex flex-col gap-2 sm:flex-row sm:items-start sm:justify-between">
|
||||
<div className="min-w-0 flex-1">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<span className="text-base">{icon}</span>
|
||||
<span
|
||||
className={`truncate text-sm font-medium ${
|
||||
source.enabled
|
||||
? 'text-stone-900 dark:text-neutral-100'
|
||||
: 'text-stone-400 line-through dark:text-neutral-500'
|
||||
}`}>
|
||||
{source.label}
|
||||
</span>
|
||||
{lastSync && (
|
||||
<span>
|
||||
{t('sync.lastChunk')} {lastSync}
|
||||
</span>
|
||||
)}
|
||||
{status.chunks_pending > 0 && (
|
||||
<span>
|
||||
{status.chunks_pending.toLocaleString()} {t('sync.pending')}
|
||||
</span>
|
||||
)}
|
||||
<span className="rounded-md bg-stone-100 px-1.5 py-0.5 text-[10px] font-medium text-stone-500 dark:bg-neutral-800 dark:text-neutral-400">
|
||||
{kindLabel}
|
||||
</span>
|
||||
{status && status.chunks_synced > 0 && <FreshnessPill freshness={status.freshness} />}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex shrink-0 items-center gap-2">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => onSync(source)}
|
||||
disabled={!source.enabled || isSyncing}
|
||||
title={t('sync.sync')}
|
||||
data-testid={`memory-source-sync-${source.toolkit ?? source.kind}`}
|
||||
className="inline-flex items-center gap-1 rounded-md bg-primary-500 px-3 py-1.5
|
||||
{detail && (
|
||||
<p className="mt-0.5 truncate pl-7 text-xs text-stone-400 dark:text-neutral-500">
|
||||
{detail}
|
||||
</p>
|
||||
)}
|
||||
{progress && (
|
||||
<div className="mt-2 pl-7">
|
||||
<div className="flex items-center gap-2 text-xs text-stone-500 dark:text-neutral-400">
|
||||
<span className="capitalize">{progress.stage}</span>
|
||||
{progress.percent !== null && (
|
||||
<span className="font-medium text-primary-600 dark:text-primary-400">
|
||||
{progress.percent}%
|
||||
</span>
|
||||
)}
|
||||
{progress.detail && (
|
||||
<span className="truncate text-stone-400 dark:text-neutral-500">
|
||||
{progress.detail}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<div className="mt-1 h-1.5 w-full overflow-hidden rounded-full bg-stone-200 dark:bg-neutral-700">
|
||||
<div
|
||||
className="h-full rounded-full bg-primary-500 transition-all duration-300"
|
||||
style={{
|
||||
width: `${progress.percent ?? (progress.stage === 'fetching' ? 10 : 5)}%`,
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{!progress && status && (status.chunks_synced > 0 || status.chunks_pending > 0) && (
|
||||
<div className="mt-1 flex flex-wrap items-center gap-x-3 gap-y-0.5 pl-7 text-xs text-stone-500 dark:text-neutral-400">
|
||||
<span>
|
||||
{status.chunks_synced.toLocaleString()} {t('sync.chunks')}
|
||||
</span>
|
||||
{lastSync && (
|
||||
<span>
|
||||
{t('sync.lastChunk')} {lastSync}
|
||||
</span>
|
||||
)}
|
||||
{status.chunks_pending > 0 && (
|
||||
<span>
|
||||
{status.chunks_pending.toLocaleString()} {t('sync.pending')}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex shrink-0 items-center gap-2">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => onToggleSettings(source.id)}
|
||||
title={t('memorySources.settings.button')}
|
||||
data-testid={`memory-source-settings-${source.id}`}
|
||||
aria-expanded={settingsExpanded}
|
||||
className={`rounded p-1 transition-colors focus:outline-none focus:ring-2 focus:ring-primary-200 ${
|
||||
settingsExpanded
|
||||
? 'bg-primary-100 text-primary-600 dark:bg-primary-500/20 dark:text-primary-400'
|
||||
: 'text-stone-400 hover:bg-stone-100 hover:text-stone-600 dark:text-neutral-500 dark:hover:bg-neutral-800 dark:hover:text-neutral-300'
|
||||
}`}>
|
||||
<GearIcon />
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => onSync(source)}
|
||||
disabled={!source.enabled || isSyncing}
|
||||
title={t('sync.sync')}
|
||||
data-testid={`memory-source-sync-${source.toolkit ?? source.kind}`}
|
||||
className="inline-flex items-center gap-1 rounded-md bg-primary-500 px-3 py-1.5
|
||||
text-xs font-semibold text-white shadow-sm transition-colors
|
||||
hover:bg-primary-600 disabled:cursor-not-allowed disabled:opacity-50
|
||||
focus:outline-none focus:ring-2 focus:ring-primary-200">
|
||||
{isSyncing ? <Spinner /> : <SyncIcon />}
|
||||
{isSyncing ? t('sync.syncing') : t('sync.sync')}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => onBuild(source)}
|
||||
disabled={!source.enabled || isBuilding || isSyncing}
|
||||
title={t('memorySources.build.title')}
|
||||
className="inline-flex items-center gap-1 rounded-md border border-primary-300
|
||||
{isSyncing ? <Spinner /> : <SyncIcon />}
|
||||
{isSyncing ? t('sync.syncing') : t('sync.sync')}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => onBuild(source)}
|
||||
disabled={!source.enabled || isBuilding || isSyncing}
|
||||
title={t('memorySources.build.title')}
|
||||
className="inline-flex items-center gap-1 rounded-md border border-primary-300
|
||||
bg-white px-3 py-1.5 text-xs font-semibold text-primary-600
|
||||
shadow-sm transition-colors hover:bg-primary-50
|
||||
disabled:cursor-not-allowed disabled:opacity-50
|
||||
dark:border-primary-500/30 dark:bg-neutral-900 dark:text-primary-400
|
||||
dark:hover:bg-primary-500/10
|
||||
focus:outline-none focus:ring-2 focus:ring-primary-200">
|
||||
{isBuilding ? <Spinner /> : <BuildIcon />}
|
||||
{isBuilding ? t('memorySources.build.building') : t('memorySources.build.title')}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => onToggle(source)}
|
||||
title={source.enabled ? t('memorySources.disable') : t('memorySources.enable')}
|
||||
className={`relative h-5 w-9 rounded-full transition-colors ${
|
||||
source.enabled ? 'bg-primary-500' : 'bg-stone-300 dark:bg-neutral-600'
|
||||
}`}>
|
||||
<span
|
||||
className={`absolute top-0.5 h-4 w-4 rounded-full bg-white shadow transition-transform ${
|
||||
source.enabled ? 'left-[18px]' : 'left-0.5'
|
||||
}`}
|
||||
/>
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => onRemove(source)}
|
||||
title={t('memorySources.remove')}
|
||||
className="rounded p-1 text-stone-400 transition-colors hover:bg-coral-50
|
||||
{isBuilding ? <Spinner /> : <BuildIcon />}
|
||||
{isBuilding ? t('memorySources.build.building') : t('memorySources.build.title')}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => onToggle(source)}
|
||||
title={source.enabled ? t('memorySources.disable') : t('memorySources.enable')}
|
||||
className={`relative h-5 w-9 rounded-full transition-colors ${
|
||||
source.enabled ? 'bg-primary-500' : 'bg-stone-300 dark:bg-neutral-600'
|
||||
}`}>
|
||||
<span
|
||||
className={`absolute top-0.5 h-4 w-4 rounded-full bg-white shadow transition-transform ${
|
||||
source.enabled ? 'left-[18px]' : 'left-0.5'
|
||||
}`}
|
||||
/>
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => onRemove(source)}
|
||||
title={t('memorySources.remove')}
|
||||
className="rounded p-1 text-stone-400 transition-colors hover:bg-coral-50
|
||||
hover:text-coral-600 dark:text-neutral-500 dark:hover:bg-coral-500/10
|
||||
dark:hover:text-coral-400">
|
||||
<TrashIcon />
|
||||
</button>
|
||||
<TrashIcon />
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
{settingsExpanded && (
|
||||
<SourceSettingsPanel
|
||||
source={source}
|
||||
syncedCount={status?.chunks_synced}
|
||||
onSaved={onSettingsSaved}
|
||||
onToast={onToast}
|
||||
/>
|
||||
)}
|
||||
</li>
|
||||
);
|
||||
}
|
||||
@@ -573,3 +681,38 @@ function Spinner() {
|
||||
</svg>
|
||||
);
|
||||
}
|
||||
|
||||
function GearIcon() {
|
||||
return (
|
||||
<svg
|
||||
width="14"
|
||||
height="14"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
strokeWidth="1.8"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
aria-hidden="true">
|
||||
<circle cx="12" cy="12" r="3" />
|
||||
<path d="M19.4 15a1.65 1.65 0 00.33 1.82l.06.06a2 2 0 010 2.83 2 2 0 01-2.83 0l-.06-.06a1.65 1.65 0 00-1.82-.33 1.65 1.65 0 00-1 1.51V21a2 2 0 01-4 0v-.09A1.65 1.65 0 009 19.4a1.65 1.65 0 00-1.82.33l-.06.06a2 2 0 01-2.83-2.83l.06-.06A1.65 1.65 0 004.68 15a1.65 1.65 0 00-1.51-1H3a2 2 0 010-4h.09A1.65 1.65 0 004.6 9a1.65 1.65 0 00-.33-1.82l-.06-.06a2 2 0 012.83-2.83l.06.06A1.65 1.65 0 009 4.68a1.65 1.65 0 001-1.51V3a2 2 0 014 0v.09a1.65 1.65 0 001 1.51 1.65 1.65 0 001.82-.33l.06-.06a2 2 0 012.83 2.83l-.06.06A1.65 1.65 0 0019.4 9a1.65 1.65 0 001.51 1H21a2 2 0 010 4h-.09a1.65 1.65 0 00-1.51 1z" />
|
||||
</svg>
|
||||
);
|
||||
}
|
||||
|
||||
function AllInIcon() {
|
||||
return (
|
||||
<svg
|
||||
width="12"
|
||||
height="12"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
strokeWidth="2"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
aria-hidden="true">
|
||||
<path d="M13 2L3 14h9l-1 8 10-12h-9l1-8z" />
|
||||
</svg>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
/**
|
||||
* Expandable per-source sync-settings panel.
|
||||
*
|
||||
* Rendered inline below a SourceRow when the gear icon is clicked.
|
||||
* Shows only the limit fields relevant to the source's kind, seeded from
|
||||
* the current entry values. Empty input = omit the field on save (meaning
|
||||
* "use backend default / unlimited").
|
||||
*
|
||||
* On Save calls `updateMemorySource` and notifies the parent via
|
||||
* `onSaved` with the updated entry.
|
||||
*/
|
||||
import { useCallback, useState } from 'react';
|
||||
|
||||
import { useT } from '../../lib/i18n/I18nContext';
|
||||
import {
|
||||
type MemorySourceEntry,
|
||||
type SourceKind,
|
||||
updateMemorySource,
|
||||
} from '../../services/memorySourcesService';
|
||||
|
||||
// Which limit fields are relevant per kind. Order determines display order.
|
||||
// Only caps that are actually enforced at sync time are surfaced — the
|
||||
// per-sync token/cost budgets are not yet wired into the sync loop, so they
|
||||
// are intentionally omitted here.
|
||||
const KIND_FIELDS: Record<SourceKind, Array<keyof LimitFields>> = {
|
||||
composio: ['sync_depth_days', 'max_items'],
|
||||
github_repo: ['max_prs', 'max_issues', 'max_commits', 'sync_depth_days'],
|
||||
rss_feed: ['max_items', 'sync_depth_days'],
|
||||
twitter_query: ['since_days'],
|
||||
web_page: ['sync_depth_days'],
|
||||
folder: ['sync_depth_days'],
|
||||
};
|
||||
|
||||
// i18n key for each field label
|
||||
const FIELD_LABEL_KEYS: Record<keyof LimitFields, string> = {
|
||||
max_prs: 'memorySources.settings.maxPrs',
|
||||
max_issues: 'memorySources.settings.maxIssues',
|
||||
max_commits: 'memorySources.settings.maxCommits',
|
||||
max_items: 'memorySources.settings.maxItems',
|
||||
since_days: 'memorySources.settings.sinceDays',
|
||||
sync_depth_days: 'memorySources.settings.syncDepthDays',
|
||||
};
|
||||
|
||||
type LimitFields = Pick<
|
||||
MemorySourceEntry,
|
||||
'max_prs' | 'max_issues' | 'max_commits' | 'max_items' | 'since_days' | 'sync_depth_days'
|
||||
>;
|
||||
|
||||
// Item-count caps where a "Maxed" badge is meaningful (synced count vs cap).
|
||||
// Time-window (sync_depth_days/since_days) and budget (tokens/cost) caps don't
|
||||
// map to a chunk count, so they never show "Maxed".
|
||||
const COUNT_FIELDS = new Set<keyof LimitFields>([
|
||||
'max_items',
|
||||
'max_prs',
|
||||
'max_issues',
|
||||
'max_commits',
|
||||
]);
|
||||
|
||||
interface SourceSettingsPanelProps {
|
||||
source: MemorySourceEntry;
|
||||
/** Chunks already synced for this source — drives the "Maxed" badge. */
|
||||
syncedCount?: number;
|
||||
onSaved: (updated: MemorySourceEntry) => void;
|
||||
onToast?: (toast: { type: 'success' | 'error'; title: string; message?: string }) => void;
|
||||
}
|
||||
|
||||
export function SourceSettingsPanel({
|
||||
source,
|
||||
syncedCount,
|
||||
onSaved,
|
||||
onToast,
|
||||
}: SourceSettingsPanelProps) {
|
||||
const { t } = useT();
|
||||
const fields = KIND_FIELDS[source.kind] ?? [];
|
||||
|
||||
// Hold each field as a string so inputs can be freely edited. Seeded from the
|
||||
// entry's stored cap (fetched from the backend — the single source of truth;
|
||||
// the caps migration writes conservative defaults onto the entry). An empty
|
||||
// field genuinely means "no cap / unlimited".
|
||||
const [values, setValues] = useState<Record<string, string>>(() => {
|
||||
const init: Record<string, string> = {};
|
||||
for (const f of fields) {
|
||||
const stored = source[f as keyof MemorySourceEntry];
|
||||
init[f] = stored != null ? String(stored) : '';
|
||||
}
|
||||
return init;
|
||||
});
|
||||
|
||||
const [saving, setSaving] = useState(false);
|
||||
|
||||
const handleChange = useCallback((field: string, value: string) => {
|
||||
setValues(prev => ({ ...prev, [field]: value }));
|
||||
}, []);
|
||||
|
||||
const handleSave = useCallback(async () => {
|
||||
setSaving(true);
|
||||
try {
|
||||
const patch: Partial<LimitFields> = {};
|
||||
for (const f of fields) {
|
||||
const raw = values[f];
|
||||
if (raw !== '' && raw !== undefined) {
|
||||
const parsed = Number(raw);
|
||||
if (!Number.isFinite(parsed) || !Number.isInteger(parsed) || parsed < 0) {
|
||||
onToast?.({ type: 'error', title: t('memorySources.settings.saveFailed') });
|
||||
return;
|
||||
}
|
||||
(patch as Record<string, number>)[f] = parsed;
|
||||
}
|
||||
// Empty string → omit from patch (backend treats absence as "default")
|
||||
}
|
||||
const updated = await updateMemorySource(source.id, patch);
|
||||
onSaved(updated);
|
||||
onToast?.({ type: 'success', title: t('memorySources.settings.saved') });
|
||||
} catch (err) {
|
||||
onToast?.({
|
||||
type: 'error',
|
||||
title: t('memorySources.settings.saveFailed'),
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
} finally {
|
||||
setSaving(false);
|
||||
}
|
||||
}, [fields, source.id, values, onSaved, onToast, t]);
|
||||
|
||||
if (fields.length === 0) return null;
|
||||
|
||||
// Display name for the tooltip — the toolkit slug (title-cased) for Composio
|
||||
// sources, else the source label.
|
||||
const toolkitName = source.toolkit
|
||||
? source.toolkit.charAt(0).toUpperCase() + source.toolkit.slice(1)
|
||||
: source.label;
|
||||
const unlimitedTooltip = t('memorySources.settings.unlimitedTooltip').replace(
|
||||
'{toolkit}',
|
||||
toolkitName
|
||||
);
|
||||
|
||||
return (
|
||||
<div
|
||||
className="mt-2 ml-7 rounded-lg border border-stone-200 bg-stone-50 p-3 dark:border-neutral-700 dark:bg-neutral-800/60"
|
||||
data-testid={`source-settings-panel-${source.id}`}>
|
||||
<p className="mb-2 text-xs font-semibold text-stone-600 dark:text-neutral-300">
|
||||
{t('memorySources.settings.title')}
|
||||
</p>
|
||||
<div className="grid grid-cols-1 gap-3 sm:grid-cols-2">
|
||||
{fields.map(field => {
|
||||
const cap = Number(values[field]);
|
||||
const isUnlimited = (values[field] ?? '') === '';
|
||||
const isMaxed =
|
||||
COUNT_FIELDS.has(field) &&
|
||||
!isUnlimited &&
|
||||
Number.isFinite(cap) &&
|
||||
typeof syncedCount === 'number' &&
|
||||
syncedCount >= cap;
|
||||
return (
|
||||
<div key={field}>
|
||||
<label
|
||||
htmlFor={`src-setting-${source.id}-${field}`}
|
||||
className="mb-0.5 flex items-center gap-1.5 text-xs font-medium text-stone-600 dark:text-neutral-400">
|
||||
{t(FIELD_LABEL_KEYS[field])}
|
||||
{isMaxed && (
|
||||
<span className="rounded bg-amber-100 px-1.5 py-0.5 text-[10px] font-semibold uppercase tracking-wide text-amber-700 dark:bg-amber-500/20 dark:text-amber-300">
|
||||
{t('memorySources.settings.maxed')}
|
||||
</span>
|
||||
)}
|
||||
{isUnlimited && (
|
||||
<span
|
||||
className="inline-flex cursor-help text-stone-400 dark:text-neutral-500"
|
||||
title={unlimitedTooltip}
|
||||
aria-label={unlimitedTooltip}>
|
||||
<InfoIcon />
|
||||
</span>
|
||||
)}
|
||||
</label>
|
||||
<input
|
||||
id={`src-setting-${source.id}-${field}`}
|
||||
type="number"
|
||||
min={0}
|
||||
step={1}
|
||||
value={values[field] ?? ''}
|
||||
onChange={e => handleChange(field, e.target.value)}
|
||||
placeholder={t('memorySources.settings.unlimited')}
|
||||
className="w-full rounded-md border border-stone-200 bg-white px-2.5 py-1.5 text-xs font-mono
|
||||
text-stone-800 placeholder:text-stone-400
|
||||
dark:border-neutral-600 dark:bg-neutral-900 dark:text-neutral-200
|
||||
dark:placeholder:text-neutral-500
|
||||
focus:outline-none focus:ring-2 focus:ring-primary-200"
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
<div className="mt-3 flex justify-end">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => void handleSave()}
|
||||
disabled={saving}
|
||||
className="inline-flex items-center gap-1 rounded-md bg-primary-600 px-3 py-1.5
|
||||
text-xs font-semibold text-white shadow-sm transition-colors
|
||||
hover:bg-primary-500 disabled:cursor-not-allowed disabled:opacity-50
|
||||
focus:outline-none focus:ring-2 focus:ring-primary-200">
|
||||
{saving ? t('memorySources.settings.saving') : t('memorySources.settings.save')}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function InfoIcon() {
|
||||
return (
|
||||
<svg
|
||||
width="13"
|
||||
height="13"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
strokeWidth="2"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
aria-hidden="true">
|
||||
<circle cx="12" cy="12" r="10" />
|
||||
<path d="M12 16v-4M12 8h.01" />
|
||||
</svg>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,322 @@
|
||||
/**
|
||||
* Unit tests for MemorySourcesRegistry — All In button, gear/settings panel,
|
||||
* per-kind field visibility, Save, and existing toggle behaviour.
|
||||
*/
|
||||
import { fireEvent, screen, waitFor, within } from '@testing-library/react';
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import * as service from '../../../services/memorySourcesService';
|
||||
import type { MemorySourceEntry } from '../../../services/memorySourcesService';
|
||||
import { renderWithProviders } from '../../../test/test-utils';
|
||||
import { MemorySourcesRegistry } from '../MemorySourcesRegistry';
|
||||
|
||||
// Mock the entire service so we don't hit RPC
|
||||
vi.mock('../../../services/memorySourcesService', async () => {
|
||||
const actual = await vi.importActual<typeof import('../../../services/memorySourcesService')>(
|
||||
'../../../services/memorySourcesService'
|
||||
);
|
||||
return {
|
||||
...actual,
|
||||
listMemorySources: vi.fn(),
|
||||
memorySourcesStatusList: vi.fn(),
|
||||
updateMemorySource: vi.fn(),
|
||||
removeMemorySource: vi.fn(),
|
||||
syncMemorySource: vi.fn(),
|
||||
applyAllIn: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
// Mock tauriCommands/memoryTree — not needed in these tests
|
||||
vi.mock('../../../utils/tauriCommands/memoryTree', () => ({
|
||||
memoryTreeFlushSource: vi.fn().mockResolvedValue({ seals_fired: 0 }),
|
||||
}));
|
||||
|
||||
const mockedList = vi.mocked(service.listMemorySources);
|
||||
const mockedStatus = vi.mocked(service.memorySourcesStatusList);
|
||||
const mockedUpdate = vi.mocked(service.updateMemorySource);
|
||||
const mockedApplyAllIn = vi.mocked(service.applyAllIn);
|
||||
|
||||
function makeSource(overrides: Partial<MemorySourceEntry> = {}): MemorySourceEntry {
|
||||
return {
|
||||
id: 'src_1',
|
||||
kind: 'github_repo',
|
||||
label: 'My Repo',
|
||||
enabled: true,
|
||||
url: 'https://github.com/org/repo',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function setup(sources: MemorySourceEntry[] = [makeSource()]) {
|
||||
mockedList.mockResolvedValue(sources);
|
||||
mockedStatus.mockResolvedValue([]);
|
||||
const onToast = vi.fn();
|
||||
const result = renderWithProviders(
|
||||
<MemorySourcesRegistry onToast={onToast} pollIntervalMs={0} />,
|
||||
{}
|
||||
);
|
||||
return { ...result, onToast };
|
||||
}
|
||||
|
||||
describe('MemorySourcesRegistry', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Basic render
|
||||
// -------------------------------------------------------------------------
|
||||
it('renders loaded sources list', async () => {
|
||||
setup([makeSource({ label: 'Work Repo' })]);
|
||||
await screen.findByText('Work Repo');
|
||||
expect(screen.getByTestId('memory-sources')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('renders empty state when no sources', async () => {
|
||||
mockedList.mockResolvedValue([]);
|
||||
mockedStatus.mockResolvedValue([]);
|
||||
renderWithProviders(<MemorySourcesRegistry pollIntervalMs={0} />);
|
||||
await screen.findByText(/no memory sources/i);
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Toggle (existing behaviour)
|
||||
// -------------------------------------------------------------------------
|
||||
it('toggle calls updateMemorySource and flips state', async () => {
|
||||
const source = makeSource({ enabled: true });
|
||||
mockedUpdate.mockResolvedValue({ ...source, enabled: false });
|
||||
setup([source]);
|
||||
await screen.findByText('My Repo');
|
||||
|
||||
const toggle = screen.getByTitle(/disable/i);
|
||||
fireEvent.click(toggle);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(mockedUpdate).toHaveBeenCalledWith('src_1', { enabled: false });
|
||||
});
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// All In button
|
||||
// -------------------------------------------------------------------------
|
||||
it('All In button is rendered in the header', async () => {
|
||||
setup();
|
||||
await screen.findByText('My Repo');
|
||||
expect(screen.getByTestId('all-in-button')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('clicking All In opens a confirmation modal', async () => {
|
||||
setup();
|
||||
await screen.findByText('My Repo');
|
||||
|
||||
fireEvent.click(screen.getByTestId('all-in-button'));
|
||||
|
||||
// The modal should appear
|
||||
await screen.findByText('Go All In?');
|
||||
expect(
|
||||
screen.getByText(/This enables every memory source and removes all sync limits/i)
|
||||
).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('cancelling All In modal closes it without calling applyAllIn', async () => {
|
||||
setup();
|
||||
await screen.findByText('My Repo');
|
||||
|
||||
fireEvent.click(screen.getByTestId('all-in-button'));
|
||||
await screen.findByText('Go All In?');
|
||||
|
||||
// Click the No / cancel button
|
||||
fireEvent.click(screen.getByText('No'));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.queryByText('Go All In?')).not.toBeInTheDocument();
|
||||
});
|
||||
expect(mockedApplyAllIn).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('confirming All In calls applyAllIn, updates sources, and shows success toast', async () => {
|
||||
const updatedSrc = makeSource({ id: 'src_2', label: 'New Repo', enabled: true });
|
||||
mockedApplyAllIn.mockResolvedValue({ sources: [updatedSrc], sync_triggered: 1 });
|
||||
|
||||
const { onToast } = setup();
|
||||
await screen.findByText('My Repo');
|
||||
|
||||
fireEvent.click(screen.getByTestId('all-in-button'));
|
||||
await screen.findByText('Go All In?');
|
||||
|
||||
fireEvent.click(screen.getByText('Yes'));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(mockedApplyAllIn).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
await waitFor(() => {
|
||||
expect(onToast).toHaveBeenCalledWith(expect.objectContaining({ type: 'success' }));
|
||||
});
|
||||
|
||||
// Modal should close
|
||||
await waitFor(() => {
|
||||
expect(screen.queryByText('Go All In?')).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
it('All In failure shows error toast', async () => {
|
||||
mockedApplyAllIn.mockRejectedValue(new Error('RPC error'));
|
||||
|
||||
const { onToast } = setup();
|
||||
await screen.findByText('My Repo');
|
||||
|
||||
fireEvent.click(screen.getByTestId('all-in-button'));
|
||||
await screen.findByText('Go All In?');
|
||||
|
||||
fireEvent.click(screen.getByText('Yes'));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(onToast).toHaveBeenCalledWith(expect.objectContaining({ type: 'error' }));
|
||||
});
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Gear / settings panel — toggling
|
||||
// -------------------------------------------------------------------------
|
||||
it('gear button renders for each source row', async () => {
|
||||
setup([makeSource({ id: 'src_1' }), makeSource({ id: 'src_2', label: 'Second' })]);
|
||||
await screen.findByText('My Repo');
|
||||
|
||||
expect(screen.getByTestId('memory-source-settings-src_1')).toBeInTheDocument();
|
||||
expect(screen.getByTestId('memory-source-settings-src_2')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('clicking gear expands the settings panel for that source', async () => {
|
||||
setup([makeSource({ id: 'src_1', kind: 'github_repo' })]);
|
||||
await screen.findByText('My Repo');
|
||||
|
||||
expect(screen.queryByTestId('source-settings-panel-src_1')).not.toBeInTheDocument();
|
||||
|
||||
fireEvent.click(screen.getByTestId('memory-source-settings-src_1'));
|
||||
|
||||
expect(screen.getByTestId('source-settings-panel-src_1')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('clicking gear again collapses the settings panel', async () => {
|
||||
setup([makeSource({ id: 'src_1', kind: 'github_repo' })]);
|
||||
await screen.findByText('My Repo');
|
||||
|
||||
const gearBtn = screen.getByTestId('memory-source-settings-src_1');
|
||||
fireEvent.click(gearBtn);
|
||||
expect(screen.getByTestId('source-settings-panel-src_1')).toBeInTheDocument();
|
||||
|
||||
fireEvent.click(gearBtn);
|
||||
expect(screen.queryByTestId('source-settings-panel-src_1')).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Settings panel — field visibility per kind
|
||||
// -------------------------------------------------------------------------
|
||||
it('github_repo settings panel shows max_prs, max_issues, max_commits, sync_depth_days', async () => {
|
||||
setup([makeSource({ id: 'src_1', kind: 'github_repo' })]);
|
||||
await screen.findByText('My Repo');
|
||||
|
||||
fireEvent.click(screen.getByTestId('memory-source-settings-src_1'));
|
||||
|
||||
const panel = screen.getByTestId('source-settings-panel-src_1');
|
||||
expect(within(panel).getByLabelText(/max pull requests/i)).toBeInTheDocument();
|
||||
expect(within(panel).getByLabelText(/max issues/i)).toBeInTheDocument();
|
||||
expect(within(panel).getByLabelText(/max commits/i)).toBeInTheDocument();
|
||||
expect(within(panel).getByLabelText(/sync depth/i)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('composio settings panel shows sync_depth_days and max_items but NOT max_prs', async () => {
|
||||
setup([makeSource({ id: 'src_1', kind: 'composio', toolkit: 'github' })]);
|
||||
await screen.findByText('My Repo');
|
||||
|
||||
fireEvent.click(screen.getByTestId('memory-source-settings-src_1'));
|
||||
|
||||
const panel = screen.getByTestId('source-settings-panel-src_1');
|
||||
expect(within(panel).getByLabelText(/max items/i)).toBeInTheDocument();
|
||||
expect(within(panel).getByLabelText(/sync depth/i)).toBeInTheDocument();
|
||||
expect(within(panel).queryByLabelText(/max pull requests/i)).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('rss_feed settings panel shows max_items and sync_depth_days but NOT max_prs', async () => {
|
||||
setup([makeSource({ id: 'src_1', kind: 'rss_feed', url: 'https://example.com/feed.xml' })]);
|
||||
await screen.findByText('My Repo');
|
||||
|
||||
fireEvent.click(screen.getByTestId('memory-source-settings-src_1'));
|
||||
|
||||
const panel = screen.getByTestId('source-settings-panel-src_1');
|
||||
expect(within(panel).getByLabelText(/max items/i)).toBeInTheDocument();
|
||||
expect(within(panel).getByLabelText(/sync depth/i)).toBeInTheDocument();
|
||||
expect(within(panel).queryByLabelText(/max pull requests/i)).not.toBeInTheDocument();
|
||||
expect(within(panel).queryByLabelText(/max commits/i)).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Settings panel — Save
|
||||
// -------------------------------------------------------------------------
|
||||
it('Save in settings panel calls updateMemorySource with numeric patch', async () => {
|
||||
const source = makeSource({ id: 'src_1', kind: 'github_repo' });
|
||||
const updated = { ...source, max_prs: 50 };
|
||||
mockedUpdate.mockResolvedValue(updated);
|
||||
|
||||
const { onToast } = setup([source]);
|
||||
await screen.findByText('My Repo');
|
||||
|
||||
fireEvent.click(screen.getByTestId('memory-source-settings-src_1'));
|
||||
|
||||
const panel = screen.getByTestId('source-settings-panel-src_1');
|
||||
const maxPrsInput = within(panel).getByLabelText(/max pull requests/i);
|
||||
fireEvent.change(maxPrsInput, { target: { value: '50' } });
|
||||
|
||||
const saveBtn = within(panel).getByText('Save');
|
||||
fireEvent.click(saveBtn);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(mockedUpdate).toHaveBeenCalledWith('src_1', expect.objectContaining({ max_prs: 50 }));
|
||||
});
|
||||
|
||||
await waitFor(() => {
|
||||
expect(onToast).toHaveBeenCalledWith(expect.objectContaining({ type: 'success' }));
|
||||
});
|
||||
});
|
||||
|
||||
it('empty input is omitted from the save patch (not sent as 0)', async () => {
|
||||
const source = makeSource({ id: 'src_1', kind: 'github_repo', max_prs: 10 });
|
||||
mockedUpdate.mockResolvedValue(source);
|
||||
|
||||
setup([source]);
|
||||
await screen.findByText('My Repo');
|
||||
|
||||
fireEvent.click(screen.getByTestId('memory-source-settings-src_1'));
|
||||
|
||||
const panel = screen.getByTestId('source-settings-panel-src_1');
|
||||
const maxPrsInput = within(panel).getByLabelText(/max pull requests/i);
|
||||
|
||||
// Clear the field
|
||||
fireEvent.change(maxPrsInput, { target: { value: '' } });
|
||||
|
||||
fireEvent.click(within(panel).getByText('Save'));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(mockedUpdate).toHaveBeenCalledWith(
|
||||
'src_1',
|
||||
expect.not.objectContaining({ max_prs: expect.anything() })
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it('Save failure shows error toast', async () => {
|
||||
mockedUpdate.mockRejectedValue(new Error('Save failed'));
|
||||
|
||||
const { onToast } = setup([makeSource({ kind: 'github_repo' })]);
|
||||
await screen.findByText('My Repo');
|
||||
|
||||
fireEvent.click(screen.getByTestId('memory-source-settings-src_1'));
|
||||
const panel = screen.getByTestId('source-settings-panel-src_1');
|
||||
fireEvent.click(within(panel).getByText('Save'));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(onToast).toHaveBeenCalledWith(expect.objectContaining({ type: 'error' }));
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1982,6 +1982,32 @@ const messages: TranslationMap = {
|
||||
'memorySources.build.successTitle': 'تم بناء الشجرة',
|
||||
'memorySources.build.failedTitle': 'فشل البناء',
|
||||
'memorySources.build.sealsMessage': 'تم إتمام عملية الختم',
|
||||
'memorySources.allIn.button': 'الكل',
|
||||
'memorySources.allIn.title': 'هل تريد تفعيل الكل؟',
|
||||
'memorySources.allIn.message':
|
||||
'سيؤدي هذا إلى تمكين جميع مصادر الذاكرة وإزالة جميع قيود المزامنة. يبني أغنى رسم بياني للذاكرة، لكنه قد يستهلك المزيد من الاعتمادات.',
|
||||
'memorySources.allIn.confirm': 'نعم',
|
||||
'memorySources.allIn.cancel': 'لا',
|
||||
'memorySources.allIn.success': 'تم تمكين جميع المصادر بدون قيود. بدأت المزامنة.',
|
||||
'memorySources.allIn.failed': 'تعذّر تطبيق خيار الكل. يرجى المحاولة مرة أخرى.',
|
||||
'memorySources.settings.button': 'الإعدادات',
|
||||
'memorySources.settings.title': 'إعدادات المزامنة',
|
||||
'memorySources.settings.maxPrs': 'أقصى عدد لطلبات السحب',
|
||||
'memorySources.settings.maxIssues': 'أقصى عدد للمشكلات',
|
||||
'memorySources.settings.maxCommits': 'أقصى عدد للإيداعات',
|
||||
'memorySources.settings.maxItems': 'أقصى عدد للعناصر',
|
||||
'memorySources.settings.sinceDays': 'فترة البحث (بالأيام)',
|
||||
'memorySources.settings.syncDepthDays': 'عمق المزامنة (بالأيام)',
|
||||
'memorySources.settings.maxTokens': 'أقصى عدد للرموز لكل مزامنة',
|
||||
'memorySources.settings.maxCost': 'الحد الأقصى للتكلفة لكل مزامنة (دولار)',
|
||||
'memorySources.settings.unlimited': 'غير محدود',
|
||||
'memorySources.settings.unlimitedTooltip':
|
||||
'لقد اخترت مزامنة الحد الأقصى لـ {toolkit}. يمكنك تغيير الحدود من هنا.',
|
||||
'memorySources.settings.maxed': 'مكتمل',
|
||||
'memorySources.settings.save': 'حفظ',
|
||||
'memorySources.settings.saving': 'جارٍ الحفظ…',
|
||||
'memorySources.settings.saved': 'تم حفظ الإعدادات',
|
||||
'memorySources.settings.saveFailed': 'تعذّر حفظ الإعدادات',
|
||||
'backend.aiBackend': 'خلفية الذكاء الاصطناعي',
|
||||
'backend.cloud': 'سحابي',
|
||||
'backend.recommended': 'موصى به',
|
||||
|
||||
@@ -2021,6 +2021,32 @@ const messages: TranslationMap = {
|
||||
'memorySources.build.successTitle': 'ট্রি তৈরি হয়েছে',
|
||||
'memorySources.build.failedTitle': 'তৈরি ব্যর্থ',
|
||||
'memorySources.build.sealsMessage': 'সিল সম্পন্ন হয়েছে',
|
||||
'memorySources.allIn.button': 'সব চালু',
|
||||
'memorySources.allIn.title': 'সব সক্রিয় করবেন?',
|
||||
'memorySources.allIn.message':
|
||||
'এটি সমস্ত মেমরি উৎস সক্রিয় করবে এবং সিঙ্ক সীমা সরিয়ে দেবে। এটি সমৃদ্ধতম মেমরি গ্রাফ তৈরি করে, তবে আরও ক্রেডিট ব্যবহার করতে পারে।',
|
||||
'memorySources.allIn.confirm': 'হ্যাঁ',
|
||||
'memorySources.allIn.cancel': 'না',
|
||||
'memorySources.allIn.success': 'সব উৎস সীমা ছাড়া সক্রিয় হয়েছে। সিঙ্ক শুরু হয়েছে।',
|
||||
'memorySources.allIn.failed': 'সব চালু করা সম্ভব হয়নি। আবার চেষ্টা করুন।',
|
||||
'memorySources.settings.button': 'সেটিংস',
|
||||
'memorySources.settings.title': 'সিঙ্ক সেটিংস',
|
||||
'memorySources.settings.maxPrs': 'সর্বোচ্চ পুল রিকোয়েস্ট',
|
||||
'memorySources.settings.maxIssues': 'সর্বোচ্চ ইস্যু',
|
||||
'memorySources.settings.maxCommits': 'সর্বোচ্চ কমিট',
|
||||
'memorySources.settings.maxItems': 'সর্বোচ্চ আইটেম',
|
||||
'memorySources.settings.sinceDays': 'পূর্ববর্তী সময় (দিনে)',
|
||||
'memorySources.settings.syncDepthDays': 'সিঙ্ক গভীরতা (দিনে)',
|
||||
'memorySources.settings.maxTokens': 'প্রতি সিঙ্কে সর্বোচ্চ টোকেন',
|
||||
'memorySources.settings.maxCost': 'প্রতি সিঙ্কে সর্বোচ্চ খরচ (USD)',
|
||||
'memorySources.settings.unlimited': 'সীমাহীন',
|
||||
'memorySources.settings.unlimitedTooltip':
|
||||
'আপনি {toolkit}-এর জন্য সর্বাধিক সিঙ্ক করার অপশন বেছে নিয়েছেন। আপনি এখানে সীমা পরিবর্তন করতে পারেন।',
|
||||
'memorySources.settings.maxed': 'পূর্ণ',
|
||||
'memorySources.settings.save': 'সংরক্ষণ',
|
||||
'memorySources.settings.saving': 'সংরক্ষণ হচ্ছে…',
|
||||
'memorySources.settings.saved': 'সেটিংস সংরক্ষিত হয়েছে',
|
||||
'memorySources.settings.saveFailed': 'সেটিংস সংরক্ষণ করা সম্ভব হয়নি',
|
||||
'backend.aiBackend': 'AI ব্যাকএন্ড',
|
||||
'backend.cloud': 'ক্লাউড',
|
||||
'backend.recommended': 'প্রস্তাবিত',
|
||||
|
||||
@@ -2072,6 +2072,32 @@ const messages: TranslationMap = {
|
||||
'memorySources.build.successTitle': 'Baum erstellt',
|
||||
'memorySources.build.failedTitle': 'Erstellung fehlgeschlagen',
|
||||
'memorySources.build.sealsMessage': 'Versiegelung(en) abgeschlossen',
|
||||
'memorySources.allIn.button': 'Alles aktivieren',
|
||||
'memorySources.allIn.title': 'Alles aktivieren?',
|
||||
'memorySources.allIn.message':
|
||||
'Dadurch werden alle Speicherquellen aktiviert und alle Synchronisierungslimits entfernt. Es erstellt den reichhaltigsten Speichergraphen, kann aber mehr Credits verbrauchen.',
|
||||
'memorySources.allIn.confirm': 'Ja',
|
||||
'memorySources.allIn.cancel': 'Nein',
|
||||
'memorySources.allIn.success': 'Alle Quellen ohne Limits aktiviert. Synchronisierung gestartet.',
|
||||
'memorySources.allIn.failed': 'Konnte „Alles aktivieren" nicht anwenden. Bitte erneut versuchen.',
|
||||
'memorySources.settings.button': 'Einstellungen',
|
||||
'memorySources.settings.title': 'Synchronisierungseinstellungen',
|
||||
'memorySources.settings.maxPrs': 'Maximale Pull-Requests',
|
||||
'memorySources.settings.maxIssues': 'Maximale Issues',
|
||||
'memorySources.settings.maxCommits': 'Maximale Commits',
|
||||
'memorySources.settings.maxItems': 'Maximale Elemente',
|
||||
'memorySources.settings.sinceDays': 'Rückblick (Tage)',
|
||||
'memorySources.settings.syncDepthDays': 'Synchronisierungstiefe (Tage)',
|
||||
'memorySources.settings.maxTokens': 'Maximale Tokens pro Synchronisierung',
|
||||
'memorySources.settings.maxCost': 'Maximale Kosten pro Synchronisierung (USD)',
|
||||
'memorySources.settings.unlimited': 'Unbegrenzt',
|
||||
'memorySources.settings.unlimitedTooltip':
|
||||
'Du hast dich entschieden, das Maximum für {toolkit} zu synchronisieren. Du kannst die Limits hier ändern.',
|
||||
'memorySources.settings.maxed': 'Voll',
|
||||
'memorySources.settings.save': 'Speichern',
|
||||
'memorySources.settings.saving': 'Speichern…',
|
||||
'memorySources.settings.saved': 'Einstellungen gespeichert',
|
||||
'memorySources.settings.saveFailed': 'Einstellungen konnten nicht gespeichert werden',
|
||||
'backend.aiBackend': 'KI-Backend',
|
||||
'backend.cloud': 'Wolke',
|
||||
'backend.recommended': 'Empfohlen',
|
||||
|
||||
@@ -2351,6 +2351,32 @@ const en: TranslationMap = {
|
||||
'memorySources.build.successTitle': 'Tree built',
|
||||
'memorySources.build.failedTitle': 'Build failed',
|
||||
'memorySources.build.sealsMessage': 'seal(s) completed',
|
||||
'memorySources.allIn.button': 'All In',
|
||||
'memorySources.allIn.title': 'Go All In?',
|
||||
'memorySources.allIn.message':
|
||||
'This enables every memory source and removes all sync limits. It builds the richest memory graph, but may use more credits.',
|
||||
'memorySources.allIn.confirm': 'Yes',
|
||||
'memorySources.allIn.cancel': 'No',
|
||||
'memorySources.allIn.success': 'All sources enabled with no limits. Syncing started.',
|
||||
'memorySources.allIn.failed': 'Could not apply All In. Please try again.',
|
||||
'memorySources.settings.button': 'Settings',
|
||||
'memorySources.settings.title': 'Sync settings',
|
||||
'memorySources.settings.maxPrs': 'Max pull requests',
|
||||
'memorySources.settings.maxIssues': 'Max issues',
|
||||
'memorySources.settings.maxCommits': 'Max commits',
|
||||
'memorySources.settings.maxItems': 'Max items',
|
||||
'memorySources.settings.sinceDays': 'Lookback (days)',
|
||||
'memorySources.settings.syncDepthDays': 'Sync depth (days)',
|
||||
'memorySources.settings.maxTokens': 'Max tokens per sync',
|
||||
'memorySources.settings.maxCost': 'Max cost per sync (USD)',
|
||||
'memorySources.settings.unlimited': 'Unlimited',
|
||||
'memorySources.settings.unlimitedTooltip':
|
||||
"You've opted in to sync the maximum for {toolkit}. You can change the caps here.",
|
||||
'memorySources.settings.maxed': 'Maxed',
|
||||
'memorySources.settings.save': 'Save',
|
||||
'memorySources.settings.saving': 'Saving…',
|
||||
'memorySources.settings.saved': 'Settings saved',
|
||||
'memorySources.settings.saveFailed': 'Could not save settings',
|
||||
|
||||
// Backend
|
||||
'backend.aiBackend': 'AI Backend',
|
||||
|
||||
@@ -2064,6 +2064,33 @@ const messages: TranslationMap = {
|
||||
'memorySources.build.successTitle': 'Árbol construido',
|
||||
'memorySources.build.failedTitle': 'Construcción fallida',
|
||||
'memorySources.build.sealsMessage': 'sellado(s) completado(s)',
|
||||
'memorySources.allIn.button': 'Todo activo',
|
||||
'memorySources.allIn.title': '¿Activar todo?',
|
||||
'memorySources.allIn.message':
|
||||
'Esto activará todas las fuentes de memoria y eliminará todos los límites de sincronización. Construye el gráfico de memoria más rico, pero puede usar más créditos.',
|
||||
'memorySources.allIn.confirm': 'Sí',
|
||||
'memorySources.allIn.cancel': 'No',
|
||||
'memorySources.allIn.success':
|
||||
'Todas las fuentes activadas sin límites. Sincronización iniciada.',
|
||||
'memorySources.allIn.failed': 'No se pudo aplicar todo activo. Por favor, inténtelo de nuevo.',
|
||||
'memorySources.settings.button': 'Configuración',
|
||||
'memorySources.settings.title': 'Configuración de sincronización',
|
||||
'memorySources.settings.maxPrs': 'Máximo de pull requests',
|
||||
'memorySources.settings.maxIssues': 'Máximo de issues',
|
||||
'memorySources.settings.maxCommits': 'Máximo de commits',
|
||||
'memorySources.settings.maxItems': 'Máximo de elementos',
|
||||
'memorySources.settings.sinceDays': 'Período de búsqueda (días)',
|
||||
'memorySources.settings.syncDepthDays': 'Profundidad de sincronización (días)',
|
||||
'memorySources.settings.maxTokens': 'Máximo de tokens por sincronización',
|
||||
'memorySources.settings.maxCost': 'Costo máximo por sincronización (USD)',
|
||||
'memorySources.settings.unlimited': 'Sin límite',
|
||||
'memorySources.settings.unlimitedTooltip':
|
||||
'Has optado por sincronizar el máximo para {toolkit}. Puedes cambiar los límites aquí.',
|
||||
'memorySources.settings.maxed': 'Lleno',
|
||||
'memorySources.settings.save': 'Guardar',
|
||||
'memorySources.settings.saving': 'Guardando…',
|
||||
'memorySources.settings.saved': 'Configuración guardada',
|
||||
'memorySources.settings.saveFailed': 'No se pudo guardar la configuración',
|
||||
'backend.aiBackend': 'Backend de IA',
|
||||
'backend.cloud': 'Nube',
|
||||
'backend.recommended': 'Recomendado',
|
||||
|
||||
@@ -2070,6 +2070,32 @@ const messages: TranslationMap = {
|
||||
'memorySources.build.successTitle': 'Arbre construit',
|
||||
'memorySources.build.failedTitle': 'Échec de construction',
|
||||
'memorySources.build.sealsMessage': 'scellement(s) terminé(s)',
|
||||
'memorySources.allIn.button': 'Tout activer',
|
||||
'memorySources.allIn.title': 'Tout activer ?',
|
||||
'memorySources.allIn.message':
|
||||
'Cela activera toutes les sources de mémoire et supprimera toutes les limites de synchronisation. Cela construit le graphe de mémoire le plus riche, mais peut utiliser davantage de crédits.',
|
||||
'memorySources.allIn.confirm': 'Oui',
|
||||
'memorySources.allIn.cancel': 'Non',
|
||||
'memorySources.allIn.success': 'Toutes les sources activées sans limite. Synchronisation lancée.',
|
||||
'memorySources.allIn.failed': "Impossible d'appliquer tout activer. Veuillez réessayer.",
|
||||
'memorySources.settings.button': 'Paramètres',
|
||||
'memorySources.settings.title': 'Paramètres de synchronisation',
|
||||
'memorySources.settings.maxPrs': 'Nombre maximal de pull requests',
|
||||
'memorySources.settings.maxIssues': "Nombre maximal d'issues",
|
||||
'memorySources.settings.maxCommits': 'Nombre maximal de commits',
|
||||
'memorySources.settings.maxItems': "Nombre maximal d'éléments",
|
||||
'memorySources.settings.sinceDays': 'Période de recherche (jours)',
|
||||
'memorySources.settings.syncDepthDays': 'Profondeur de synchronisation (jours)',
|
||||
'memorySources.settings.maxTokens': 'Tokens maximum par synchronisation',
|
||||
'memorySources.settings.maxCost': 'Coût maximum par synchronisation (USD)',
|
||||
'memorySources.settings.unlimited': 'Illimité',
|
||||
'memorySources.settings.unlimitedTooltip':
|
||||
'Vous avez choisi de synchroniser le maximum pour {toolkit}. Vous pouvez modifier les limites ici.',
|
||||
'memorySources.settings.maxed': 'Plein',
|
||||
'memorySources.settings.save': 'Enregistrer',
|
||||
'memorySources.settings.saving': 'Enregistrement…',
|
||||
'memorySources.settings.saved': 'Paramètres enregistrés',
|
||||
'memorySources.settings.saveFailed': "Impossible d'enregistrer les paramètres",
|
||||
'backend.aiBackend': 'Backend IA',
|
||||
'backend.cloud': 'Cloud',
|
||||
'backend.recommended': 'Recommandé',
|
||||
|
||||
@@ -2022,6 +2022,32 @@ const messages: TranslationMap = {
|
||||
'memorySources.build.successTitle': 'ट्री बन गई',
|
||||
'memorySources.build.failedTitle': 'निर्माण विफल',
|
||||
'memorySources.build.sealsMessage': 'सील पूर्ण हुई',
|
||||
'memorySources.allIn.button': 'सब चालू',
|
||||
'memorySources.allIn.title': 'सब कुछ सक्रिय करें?',
|
||||
'memorySources.allIn.message':
|
||||
'यह सभी मेमोरी स्रोत सक्रिय करेगा और सभी सिंक सीमाएं हटा देगा। यह सबसे समृद्ध मेमोरी ग्राफ बनाता है, लेकिन अधिक क्रेडिट उपयोग कर सकता है।',
|
||||
'memorySources.allIn.confirm': 'हाँ',
|
||||
'memorySources.allIn.cancel': 'नहीं',
|
||||
'memorySources.allIn.success': 'सभी स्रोत बिना सीमा के सक्रिय हुए। सिंक शुरू हो गई।',
|
||||
'memorySources.allIn.failed': 'सब चालू नहीं किया जा सका। कृपया पुनः प्रयास करें।',
|
||||
'memorySources.settings.button': 'सेटिंग',
|
||||
'memorySources.settings.title': 'सिंक सेटिंग',
|
||||
'memorySources.settings.maxPrs': 'अधिकतम पुल रिक्वेस्ट',
|
||||
'memorySources.settings.maxIssues': 'अधिकतम इश्यू',
|
||||
'memorySources.settings.maxCommits': 'अधिकतम कमिट',
|
||||
'memorySources.settings.maxItems': 'अधिकतम आइटम',
|
||||
'memorySources.settings.sinceDays': 'पिछले दिन (दिनों में)',
|
||||
'memorySources.settings.syncDepthDays': 'सिंक गहराई (दिनों में)',
|
||||
'memorySources.settings.maxTokens': 'प्रति सिंक अधिकतम टोकन',
|
||||
'memorySources.settings.maxCost': 'प्रति सिंक अधिकतम लागत (USD)',
|
||||
'memorySources.settings.unlimited': 'असीमित',
|
||||
'memorySources.settings.unlimitedTooltip':
|
||||
'आपने {toolkit} के लिए अधिकतम सिंक करना चुना है। आप यहाँ सीमाएँ बदल सकते हैं।',
|
||||
'memorySources.settings.maxed': 'पूर्ण',
|
||||
'memorySources.settings.save': 'सहेजें',
|
||||
'memorySources.settings.saving': 'सहेजा जा रहा है…',
|
||||
'memorySources.settings.saved': 'सेटिंग सहेजी गई',
|
||||
'memorySources.settings.saveFailed': 'सेटिंग सहेजी नहीं जा सकी',
|
||||
'backend.aiBackend': 'AI बैकएंड',
|
||||
'backend.cloud': 'क्लाउड',
|
||||
'backend.recommended': 'सुझावित',
|
||||
|
||||
@@ -2025,6 +2025,32 @@ const messages: TranslationMap = {
|
||||
'memorySources.build.successTitle': 'Pohon dibangun',
|
||||
'memorySources.build.failedTitle': 'Pembangunan gagal',
|
||||
'memorySources.build.sealsMessage': 'penyegelan selesai',
|
||||
'memorySources.allIn.button': 'Aktifkan Semua',
|
||||
'memorySources.allIn.title': 'Aktifkan Semua?',
|
||||
'memorySources.allIn.message':
|
||||
'Ini akan mengaktifkan semua sumber memori dan menghapus semua batas sinkronisasi. Membangun grafik memori terkaya, tetapi mungkin menggunakan lebih banyak kredit.',
|
||||
'memorySources.allIn.confirm': 'Ya',
|
||||
'memorySources.allIn.cancel': 'Tidak',
|
||||
'memorySources.allIn.success': 'Semua sumber diaktifkan tanpa batas. Sinkronisasi dimulai.',
|
||||
'memorySources.allIn.failed': 'Gagal mengaktifkan semua. Silakan coba lagi.',
|
||||
'memorySources.settings.button': 'Pengaturan',
|
||||
'memorySources.settings.title': 'Pengaturan sinkronisasi',
|
||||
'memorySources.settings.maxPrs': 'Maksimal pull request',
|
||||
'memorySources.settings.maxIssues': 'Maksimal isu',
|
||||
'memorySources.settings.maxCommits': 'Maksimal commit',
|
||||
'memorySources.settings.maxItems': 'Maksimal item',
|
||||
'memorySources.settings.sinceDays': 'Periode ke belakang (hari)',
|
||||
'memorySources.settings.syncDepthDays': 'Kedalaman sinkronisasi (hari)',
|
||||
'memorySources.settings.maxTokens': 'Maksimal token per sinkronisasi',
|
||||
'memorySources.settings.maxCost': 'Biaya maksimal per sinkronisasi (USD)',
|
||||
'memorySources.settings.unlimited': 'Tanpa batas',
|
||||
'memorySources.settings.unlimitedTooltip':
|
||||
'Anda memilih menyinkronkan maksimum untuk {toolkit}. Anda dapat mengubah batas di sini.',
|
||||
'memorySources.settings.maxed': 'Penuh',
|
||||
'memorySources.settings.save': 'Simpan',
|
||||
'memorySources.settings.saving': 'Menyimpan…',
|
||||
'memorySources.settings.saved': 'Pengaturan tersimpan',
|
||||
'memorySources.settings.saveFailed': 'Gagal menyimpan pengaturan',
|
||||
'backend.aiBackend': 'Backend AI',
|
||||
'backend.cloud': 'Awan',
|
||||
'backend.recommended': 'Direkomendasikan',
|
||||
|
||||
@@ -2054,6 +2054,33 @@ const messages: TranslationMap = {
|
||||
'memorySources.build.successTitle': 'Albero costruito',
|
||||
'memorySources.build.failedTitle': 'Costruzione fallita',
|
||||
'memorySources.build.sealsMessage': 'sigillatura completata',
|
||||
'memorySources.allIn.button': 'Tutto attivo',
|
||||
'memorySources.allIn.title': 'Attivare tutto?',
|
||||
'memorySources.allIn.message':
|
||||
'Questo attiverà tutte le sorgenti di memoria e rimuoverà tutti i limiti di sincronizzazione. Costruisce il grafico di memoria più ricco, ma potrebbe utilizzare più crediti.',
|
||||
'memorySources.allIn.confirm': 'Sì',
|
||||
'memorySources.allIn.cancel': 'No',
|
||||
'memorySources.allIn.success':
|
||||
'Tutte le sorgenti attivate senza limiti. Sincronizzazione avviata.',
|
||||
'memorySources.allIn.failed': 'Impossibile attivare tutto. Riprova.',
|
||||
'memorySources.settings.button': 'Impostazioni',
|
||||
'memorySources.settings.title': 'Impostazioni di sincronizzazione',
|
||||
'memorySources.settings.maxPrs': 'Numero massimo di pull request',
|
||||
'memorySources.settings.maxIssues': 'Numero massimo di issue',
|
||||
'memorySources.settings.maxCommits': 'Numero massimo di commit',
|
||||
'memorySources.settings.maxItems': 'Numero massimo di elementi',
|
||||
'memorySources.settings.sinceDays': 'Periodo di ricerca (giorni)',
|
||||
'memorySources.settings.syncDepthDays': 'Profondità di sincronizzazione (giorni)',
|
||||
'memorySources.settings.maxTokens': 'Token massimi per sincronizzazione',
|
||||
'memorySources.settings.maxCost': 'Costo massimo per sincronizzazione (USD)',
|
||||
'memorySources.settings.unlimited': 'Illimitato',
|
||||
'memorySources.settings.unlimitedTooltip':
|
||||
'Hai scelto di sincronizzare il massimo per {toolkit}. Puoi modificare i limiti qui.',
|
||||
'memorySources.settings.maxed': 'Pieno',
|
||||
'memorySources.settings.save': 'Salva',
|
||||
'memorySources.settings.saving': 'Salvataggio…',
|
||||
'memorySources.settings.saved': 'Impostazioni salvate',
|
||||
'memorySources.settings.saveFailed': 'Impossibile salvare le impostazioni',
|
||||
'backend.aiBackend': 'Backend AI',
|
||||
'backend.cloud': 'Cloud',
|
||||
'backend.recommended': 'Consigliato',
|
||||
|
||||
@@ -1999,6 +1999,32 @@ const messages: TranslationMap = {
|
||||
'memorySources.build.successTitle': '트리 빌드 완료',
|
||||
'memorySources.build.failedTitle': '빌드 실패',
|
||||
'memorySources.build.sealsMessage': '실링 완료',
|
||||
'memorySources.allIn.button': '전체 활성화',
|
||||
'memorySources.allIn.title': '전체 활성화할까요?',
|
||||
'memorySources.allIn.message':
|
||||
'모든 메모리 소스를 활성화하고 동기화 제한을 제거합니다. 가장 풍부한 메모리 그래프를 구축하지만 크레딧을 더 사용할 수 있습니다.',
|
||||
'memorySources.allIn.confirm': '예',
|
||||
'memorySources.allIn.cancel': '아니오',
|
||||
'memorySources.allIn.success': '제한 없이 모든 소스가 활성화되었습니다. 동기화가 시작되었습니다.',
|
||||
'memorySources.allIn.failed': '전체 활성화를 적용하지 못했습니다. 다시 시도해 주세요.',
|
||||
'memorySources.settings.button': '설정',
|
||||
'memorySources.settings.title': '동기화 설정',
|
||||
'memorySources.settings.maxPrs': '최대 풀 리퀘스트 수',
|
||||
'memorySources.settings.maxIssues': '최대 이슈 수',
|
||||
'memorySources.settings.maxCommits': '최대 커밋 수',
|
||||
'memorySources.settings.maxItems': '최대 항목 수',
|
||||
'memorySources.settings.sinceDays': '조회 기간 (일)',
|
||||
'memorySources.settings.syncDepthDays': '동기화 깊이 (일)',
|
||||
'memorySources.settings.maxTokens': '동기화당 최대 토큰',
|
||||
'memorySources.settings.maxCost': '동기화당 최대 비용 (USD)',
|
||||
'memorySources.settings.unlimited': '무제한',
|
||||
'memorySources.settings.unlimitedTooltip':
|
||||
'{toolkit}에 대해 최대로 동기화하도록 선택했습니다. 여기에서 한도를 변경할 수 있습니다.',
|
||||
'memorySources.settings.maxed': '최대',
|
||||
'memorySources.settings.save': '저장',
|
||||
'memorySources.settings.saving': '저장 중…',
|
||||
'memorySources.settings.saved': '설정이 저장되었습니다',
|
||||
'memorySources.settings.saveFailed': '설정을 저장하지 못했습니다',
|
||||
'backend.aiBackend': 'AI 백엔드',
|
||||
'backend.cloud': '클라우드',
|
||||
'backend.recommended': '추천',
|
||||
|
||||
@@ -2043,6 +2043,33 @@ const messages: TranslationMap = {
|
||||
'memorySources.build.successTitle': 'Drzewo zbudowane',
|
||||
'memorySources.build.failedTitle': 'Budowanie nie powiodło się',
|
||||
'memorySources.build.sealsMessage': 'pieczętowanie zakończone',
|
||||
'memorySources.allIn.button': 'Włącz wszystko',
|
||||
'memorySources.allIn.title': 'Włączyć wszystko?',
|
||||
'memorySources.allIn.message':
|
||||
'Spowoduje to włączenie wszystkich źródeł pamięci i usunięcie wszystkich limitów synchronizacji. Buduje najbogatszy wykres pamięci, ale może zużywać więcej kredytów.',
|
||||
'memorySources.allIn.confirm': 'Tak',
|
||||
'memorySources.allIn.cancel': 'Nie',
|
||||
'memorySources.allIn.success':
|
||||
'Wszystkie źródła włączone bez limitów. Synchronizacja uruchomiona.',
|
||||
'memorySources.allIn.failed': 'Nie udało się włączyć wszystkiego. Spróbuj ponownie.',
|
||||
'memorySources.settings.button': 'Ustawienia',
|
||||
'memorySources.settings.title': 'Ustawienia synchronizacji',
|
||||
'memorySources.settings.maxPrs': 'Maksymalna liczba pull requestów',
|
||||
'memorySources.settings.maxIssues': 'Maksymalna liczba zgłoszeń',
|
||||
'memorySources.settings.maxCommits': 'Maksymalna liczba commitów',
|
||||
'memorySources.settings.maxItems': 'Maksymalna liczba elementów',
|
||||
'memorySources.settings.sinceDays': 'Okres wsteczny (dni)',
|
||||
'memorySources.settings.syncDepthDays': 'Głębokość synchronizacji (dni)',
|
||||
'memorySources.settings.maxTokens': 'Maksymalna liczba tokenów na synchronizację',
|
||||
'memorySources.settings.maxCost': 'Maksymalny koszt na synchronizację (USD)',
|
||||
'memorySources.settings.unlimited': 'Bez limitu',
|
||||
'memorySources.settings.unlimitedTooltip':
|
||||
'Wybrano synchronizację maksimum dla {toolkit}. Limity możesz zmienić tutaj.',
|
||||
'memorySources.settings.maxed': 'Pełny',
|
||||
'memorySources.settings.save': 'Zapisz',
|
||||
'memorySources.settings.saving': 'Zapisywanie…',
|
||||
'memorySources.settings.saved': 'Ustawienia zapisane',
|
||||
'memorySources.settings.saveFailed': 'Nie udało się zapisać ustawień',
|
||||
'backend.aiBackend': 'Backend AI',
|
||||
'backend.cloud': 'Chmura',
|
||||
'backend.recommended': 'Zalecane',
|
||||
|
||||
@@ -2062,6 +2062,32 @@ const messages: TranslationMap = {
|
||||
'memorySources.build.successTitle': 'Árvore construída',
|
||||
'memorySources.build.failedTitle': 'Falha na construção',
|
||||
'memorySources.build.sealsMessage': 'selagem concluída',
|
||||
'memorySources.allIn.button': 'Ativar tudo',
|
||||
'memorySources.allIn.title': 'Ativar tudo?',
|
||||
'memorySources.allIn.message':
|
||||
'Isso ativará todas as fontes de memória e removerá todos os limites de sincronização. Cria o grafo de memória mais rico, mas pode usar mais créditos.',
|
||||
'memorySources.allIn.confirm': 'Sim',
|
||||
'memorySources.allIn.cancel': 'Não',
|
||||
'memorySources.allIn.success': 'Todas as fontes ativadas sem limites. Sincronização iniciada.',
|
||||
'memorySources.allIn.failed': 'Não foi possível ativar tudo. Por favor, tente novamente.',
|
||||
'memorySources.settings.button': 'Configurações',
|
||||
'memorySources.settings.title': 'Configurações de sincronização',
|
||||
'memorySources.settings.maxPrs': 'Máximo de pull requests',
|
||||
'memorySources.settings.maxIssues': 'Máximo de issues',
|
||||
'memorySources.settings.maxCommits': 'Máximo de commits',
|
||||
'memorySources.settings.maxItems': 'Máximo de itens',
|
||||
'memorySources.settings.sinceDays': 'Período de busca (dias)',
|
||||
'memorySources.settings.syncDepthDays': 'Profundidade de sincronização (dias)',
|
||||
'memorySources.settings.maxTokens': 'Máximo de tokens por sincronização',
|
||||
'memorySources.settings.maxCost': 'Custo máximo por sincronização (USD)',
|
||||
'memorySources.settings.unlimited': 'Ilimitado',
|
||||
'memorySources.settings.unlimitedTooltip':
|
||||
'Você optou por sincronizar o máximo para {toolkit}. Você pode alterar os limites aqui.',
|
||||
'memorySources.settings.maxed': 'Cheio',
|
||||
'memorySources.settings.save': 'Salvar',
|
||||
'memorySources.settings.saving': 'Salvando…',
|
||||
'memorySources.settings.saved': 'Configurações salvas',
|
||||
'memorySources.settings.saveFailed': 'Não foi possível salvar as configurações',
|
||||
'backend.aiBackend': 'Backend de IA',
|
||||
'backend.cloud': 'Nuvem',
|
||||
'backend.recommended': 'Recomendado',
|
||||
|
||||
@@ -2035,6 +2035,34 @@ const messages: TranslationMap = {
|
||||
'memorySources.build.successTitle': 'Дерево построено',
|
||||
'memorySources.build.failedTitle': 'Ошибка построения',
|
||||
'memorySources.build.sealsMessage': 'запечатывание завершено',
|
||||
'memorySources.allIn.button': 'Включить всё',
|
||||
'memorySources.allIn.title': 'Включить всё?',
|
||||
'memorySources.allIn.message':
|
||||
'Это активирует все источники памяти и снимет все ограничения синхронизации. Создаёт наиболее богатый граф памяти, но может использовать больше кредитов.',
|
||||
'memorySources.allIn.confirm': 'Да',
|
||||
'memorySources.allIn.cancel': 'Нет',
|
||||
'memorySources.allIn.success':
|
||||
'Все источники активированы без ограничений. Синхронизация запущена.',
|
||||
'memorySources.allIn.failed':
|
||||
'Не удалось применить «Включить всё». Пожалуйста, попробуйте снова.',
|
||||
'memorySources.settings.button': 'Настройки',
|
||||
'memorySources.settings.title': 'Настройки синхронизации',
|
||||
'memorySources.settings.maxPrs': 'Максимум pull request',
|
||||
'memorySources.settings.maxIssues': 'Максимум задач',
|
||||
'memorySources.settings.maxCommits': 'Максимум коммитов',
|
||||
'memorySources.settings.maxItems': 'Максимум элементов',
|
||||
'memorySources.settings.sinceDays': 'Период (дней)',
|
||||
'memorySources.settings.syncDepthDays': 'Глубина синхронизации (дней)',
|
||||
'memorySources.settings.maxTokens': 'Максимум токенов за синхронизацию',
|
||||
'memorySources.settings.maxCost': 'Максимальная стоимость за синхронизацию (USD)',
|
||||
'memorySources.settings.unlimited': 'Без лимита',
|
||||
'memorySources.settings.unlimitedTooltip':
|
||||
'Вы выбрали синхронизацию максимума для {toolkit}. Лимиты можно изменить здесь.',
|
||||
'memorySources.settings.maxed': 'Заполнено',
|
||||
'memorySources.settings.save': 'Сохранить',
|
||||
'memorySources.settings.saving': 'Сохранение…',
|
||||
'memorySources.settings.saved': 'Настройки сохранены',
|
||||
'memorySources.settings.saveFailed': 'Не удалось сохранить настройки',
|
||||
'backend.aiBackend': 'AI-бэкенд',
|
||||
'backend.cloud': 'Облако',
|
||||
'backend.recommended': 'Рекомендуется',
|
||||
|
||||
@@ -1921,6 +1921,32 @@ const messages: TranslationMap = {
|
||||
'memorySources.build.successTitle': '树构建完成',
|
||||
'memorySources.build.failedTitle': '构建失败',
|
||||
'memorySources.build.sealsMessage': '密封完成',
|
||||
'memorySources.allIn.button': '全部启用',
|
||||
'memorySources.allIn.title': '全部启用?',
|
||||
'memorySources.allIn.message':
|
||||
'这将启用所有记忆来源并移除所有同步限制。可构建最丰富的记忆图谱,但可能会消耗更多积分。',
|
||||
'memorySources.allIn.confirm': '是',
|
||||
'memorySources.allIn.cancel': '否',
|
||||
'memorySources.allIn.success': '所有来源已无限制启用。同步已开始。',
|
||||
'memorySources.allIn.failed': '无法应用全部启用。请重试。',
|
||||
'memorySources.settings.button': '设置',
|
||||
'memorySources.settings.title': '同步设置',
|
||||
'memorySources.settings.maxPrs': '最大拉取请求数',
|
||||
'memorySources.settings.maxIssues': '最大问题数',
|
||||
'memorySources.settings.maxCommits': '最大提交数',
|
||||
'memorySources.settings.maxItems': '最大条目数',
|
||||
'memorySources.settings.sinceDays': '回溯天数',
|
||||
'memorySources.settings.syncDepthDays': '同步深度(天)',
|
||||
'memorySources.settings.maxTokens': '每次同步最大令牌数',
|
||||
'memorySources.settings.maxCost': '每次同步最大费用(USD)',
|
||||
'memorySources.settings.unlimited': '无限制',
|
||||
'memorySources.settings.unlimitedTooltip':
|
||||
'您已选择为 {toolkit} 同步最大数量。您可以在此处更改上限。',
|
||||
'memorySources.settings.maxed': '已满',
|
||||
'memorySources.settings.save': '保存',
|
||||
'memorySources.settings.saving': '保存中…',
|
||||
'memorySources.settings.saved': '设置已保存',
|
||||
'memorySources.settings.saveFailed': '无法保存设置',
|
||||
'backend.aiBackend': 'AI 后端',
|
||||
'backend.cloud': '云端',
|
||||
'backend.recommended': '推荐',
|
||||
|
||||
@@ -3,7 +3,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { callCoreRpc } from './coreRpcClient';
|
||||
import {
|
||||
addMemorySource,
|
||||
applyAllIn,
|
||||
listMemorySources,
|
||||
type MemorySourceEntry,
|
||||
removeMemorySource,
|
||||
SOURCE_KIND_ICONS,
|
||||
SOURCE_KIND_LABEL_KEYS,
|
||||
@@ -96,4 +98,73 @@ describe('memorySourcesService', () => {
|
||||
expect(SOURCE_KIND_ICONS[kind]).toBeTruthy();
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// applyAllIn
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
it('applyAllIn calls the correct RPC method', async () => {
|
||||
mockedCall.mockResolvedValue({
|
||||
result: {
|
||||
sources: [{ id: 'src_1', kind: 'github_repo', label: 'All Repos', enabled: true }],
|
||||
sync_triggered: 1,
|
||||
},
|
||||
logs: [],
|
||||
} as never);
|
||||
|
||||
const result = await applyAllIn();
|
||||
|
||||
expect(mockedCall).toHaveBeenCalledWith({ method: 'openhuman.memory_sources_apply_all_in' });
|
||||
expect(result.sync_triggered).toBe(1);
|
||||
expect(result.sources).toHaveLength(1);
|
||||
expect(result.sources[0].id).toBe('src_1');
|
||||
});
|
||||
|
||||
it('applyAllIn handles envelope-wrapped response', async () => {
|
||||
mockedCall.mockResolvedValue({ result: { sources: [], sync_triggered: 0 }, logs: [] } as never);
|
||||
|
||||
const result = await applyAllIn();
|
||||
expect(result.sources).toEqual([]);
|
||||
expect(result.sync_triggered).toBe(0);
|
||||
});
|
||||
|
||||
it('applyAllIn handles flat (un-wrapped) response', async () => {
|
||||
mockedCall.mockResolvedValue({
|
||||
sources: [{ id: 'src_2', kind: 'folder', label: 'Docs', enabled: true }],
|
||||
sync_triggered: 1,
|
||||
} as never);
|
||||
|
||||
const result = await applyAllIn();
|
||||
expect(result.sources).toHaveLength(1);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MemorySourceEntry interface includes all limit fields
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
it('MemorySourceEntry interface accepts all limit fields at compile-time', () => {
|
||||
// This test is a type-level assertion: if the fields are missing from the
|
||||
// interface the assignment below would fail TypeScript compilation.
|
||||
const entry: MemorySourceEntry = {
|
||||
id: 'src_1',
|
||||
kind: 'github_repo',
|
||||
label: 'Test',
|
||||
enabled: true,
|
||||
max_commits: 100,
|
||||
max_issues: 200,
|
||||
max_prs: 50,
|
||||
max_items: 500,
|
||||
since_days: 30,
|
||||
sync_depth_days: 90,
|
||||
max_tokens_per_sync: 100_000,
|
||||
max_cost_per_sync_usd: 1.5,
|
||||
};
|
||||
expect(entry.max_commits).toBe(100);
|
||||
expect(entry.max_issues).toBe(200);
|
||||
expect(entry.max_prs).toBe(50);
|
||||
expect(entry.since_days).toBe(30);
|
||||
expect(entry.sync_depth_days).toBe(90);
|
||||
expect(entry.max_tokens_per_sync).toBe(100_000);
|
||||
expect(entry.max_cost_per_sync_usd).toBe(1.5);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -31,9 +31,16 @@ export interface MemorySourceEntry {
|
||||
branch?: string;
|
||||
paths?: string[];
|
||||
query?: string;
|
||||
selector?: string;
|
||||
// Sync limit fields (all optional; omit = use backend default / unlimited)
|
||||
since_days?: number;
|
||||
max_items?: number;
|
||||
selector?: string;
|
||||
max_commits?: number;
|
||||
max_issues?: number;
|
||||
max_prs?: number;
|
||||
sync_depth_days?: number;
|
||||
max_tokens_per_sync?: number;
|
||||
max_cost_per_sync_usd?: number;
|
||||
}
|
||||
|
||||
export interface SourceItem {
|
||||
@@ -159,6 +166,25 @@ export async function syncMemorySource(sourceId: string): Promise<void> {
|
||||
});
|
||||
}
|
||||
|
||||
export interface ApplyAllInResult {
|
||||
sources: MemorySourceEntry[];
|
||||
sync_triggered: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Enables every memory source, clears all per-source sync caps, and
|
||||
* triggers a background sync for each. Equivalent to the UI "All In"
|
||||
* action. Maps to `openhuman.memory_sources_apply_all_in`.
|
||||
*/
|
||||
export async function applyAllIn(): Promise<ApplyAllInResult> {
|
||||
log('apply_all_in');
|
||||
const resp = await callCoreRpc<ApplyAllInResult>({
|
||||
method: 'openhuman.memory_sources_apply_all_in',
|
||||
});
|
||||
const data = unwrap<ApplyAllInResult>(resp);
|
||||
return { sources: data.sources ?? [], sync_triggered: data.sync_triggered ?? 0 };
|
||||
}
|
||||
|
||||
/// i18n keys for each source kind's user-visible label. Resolve via
|
||||
/// `t(SOURCE_KIND_LABEL_KEYS[kind])` in components — keeping the keys
|
||||
/// as a constant lets the dialog kind-picker render the same labels
|
||||
|
||||
@@ -451,6 +451,8 @@ async fn main() -> Result<()> {
|
||||
toolkit: conn.toolkit.clone(),
|
||||
connection_id: Some(conn.id.clone()),
|
||||
usage: Default::default(),
|
||||
max_items: None,
|
||||
sync_depth_days: None,
|
||||
};
|
||||
match run_backfill_via_search(&ctx, cli.days).await {
|
||||
Ok(outcome) => {
|
||||
@@ -549,6 +551,8 @@ async fn main() -> Result<()> {
|
||||
toolkit: conn.toolkit.clone(),
|
||||
connection_id: Some(conn.id.clone()),
|
||||
usage: Default::default(),
|
||||
max_items: None,
|
||||
sync_depth_days: None,
|
||||
};
|
||||
match provider.sync(&ctx, SyncReason::Manual).await {
|
||||
Ok(outcome) => {
|
||||
|
||||
@@ -355,6 +355,25 @@ pub(super) const CAPABILITIES: &[Capability] = &[
|
||||
status: CapabilityStatus::Beta,
|
||||
privacy: GITHUB_REPO_SOURCE,
|
||||
},
|
||||
Capability {
|
||||
id: "intelligence.memory_source_sync_controls",
|
||||
name: "Memory Source Sync Defaults & Controls",
|
||||
domain: "memory_sources",
|
||||
category: CapabilityCategory::Intelligence,
|
||||
description: "Connected memory sources are enabled by default with conservative, \
|
||||
per-kind sync caps so the first sync stays cheap (e.g. Gmail ~100 recent emails, \
|
||||
GitHub repo 10 PRs / 10 issues / 50 commits, RSS 20 items). Each source row exposes \
|
||||
an inline settings panel to adjust the limit fields that apply to its kind \
|
||||
(max_items, sync_depth_days, max_prs/issues/commits, since_days). \
|
||||
An \"All In\" action enables every source and removes the caps to build the richest \
|
||||
memory graph, then triggers a full sync. Already-connected sources are migrated to \
|
||||
the new defaults once.",
|
||||
how_to: "Intelligence > Memory Sources — toggle a source, open its gear for per-source \
|
||||
limits, or use \"All In\". Programmatic: openhuman.memory_sources_update and \
|
||||
openhuman.memory_sources_apply_all_in (RPC).",
|
||||
status: CapabilityStatus::Beta,
|
||||
privacy: LOCAL_RAW,
|
||||
},
|
||||
Capability {
|
||||
id: "intelligence.embedding_provider_config",
|
||||
name: "Configure Embedding Provider",
|
||||
|
||||
@@ -106,6 +106,7 @@ fn catalog_includes_additional_user_facing_surfaces() {
|
||||
"intelligence.embedding_provider_config",
|
||||
"intelligence.embedding_provider_test",
|
||||
"intelligence.github_repo_memory_source",
|
||||
"intelligence.memory_source_sync_controls",
|
||||
"conversation.subagent_mascots",
|
||||
] {
|
||||
assert!(
|
||||
|
||||
@@ -1264,6 +1264,8 @@ pub async fn composio_get_user_profile(
|
||||
toolkit: toolkit.clone(),
|
||||
connection_id: Some(connection_id.to_string()),
|
||||
usage: Default::default(),
|
||||
max_items: None,
|
||||
sync_depth_days: None,
|
||||
};
|
||||
|
||||
let profile = provider.fetch_user_profile(&ctx).await.map_err(|e| {
|
||||
@@ -1336,6 +1338,8 @@ pub async fn composio_refresh_all_identities(
|
||||
toolkit: toolkit.clone(),
|
||||
connection_id: Some(connection_id.clone()),
|
||||
usage: Default::default(),
|
||||
max_items: None,
|
||||
sync_depth_days: None,
|
||||
};
|
||||
|
||||
match provider.fetch_user_profile(&ctx).await {
|
||||
@@ -1433,6 +1437,8 @@ pub async fn composio_sync(
|
||||
toolkit: toolkit.clone(),
|
||||
connection_id: Some(connection_id.to_string()),
|
||||
usage: Default::default(),
|
||||
max_items: None,
|
||||
sync_depth_days: None,
|
||||
};
|
||||
let started_at_ms = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
|
||||
@@ -419,6 +419,18 @@ pub struct Config {
|
||||
|
||||
#[serde(default)]
|
||||
pub model_registry: Vec<ModelRegistryEntry>,
|
||||
|
||||
/// Migration version guard for `apply_composio_source_caps_migration`.
|
||||
///
|
||||
/// The migration runs whenever this is `< CURRENT_CAPS_MIGRATION_VERSION`
|
||||
/// (see `memory_sources::reconcile`), then is bumped to that version. Using a
|
||||
/// monotonic version (rather than a bool) lets an improved migration re-run
|
||||
/// once for installs that already ran an earlier revision. Defaults to `0`
|
||||
/// (`#[serde(default)]`); the retired `composio_source_caps_migrated` bool is
|
||||
/// silently ignored (Config does not `deny_unknown_fields`), so prior installs
|
||||
/// re-run the current migration exactly once.
|
||||
#[serde(default)]
|
||||
pub composio_source_caps_migration_version: u32,
|
||||
}
|
||||
|
||||
/// Shared default so `#[serde(default)]` and `Config::default()` stay in sync.
|
||||
@@ -738,6 +750,7 @@ impl Default for Config {
|
||||
onboarding_completed: false,
|
||||
chat_onboarding_completed: false,
|
||||
model_registry: Vec::new(),
|
||||
composio_source_caps_migration_version: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,10 +25,11 @@ pub mod sync;
|
||||
pub mod types;
|
||||
|
||||
pub use registry::{
|
||||
add_source, get_source, list_enabled_by_kind, list_sources,
|
||||
remove_composio_source_by_connection_id, remove_source, update_source, upsert_composio_source,
|
||||
MemorySourcePatch,
|
||||
add_source, apply_all_in, get_source, list_enabled_by_kind, list_sources,
|
||||
memory_sync_defaults_for_toolkit, remove_composio_source_by_connection_id, remove_source,
|
||||
update_source, upsert_composio_source, MemorySourcePatch,
|
||||
};
|
||||
pub use rpc::apply_kind_defaults;
|
||||
pub use schemas::{
|
||||
all_controller_schemas as all_memory_sources_controller_schemas,
|
||||
all_registered_controllers as all_memory_sources_registered_controllers,
|
||||
|
||||
@@ -3,11 +3,20 @@
|
||||
//! Called once at boot to ensure all active Composio sync targets have
|
||||
//! a corresponding `MemorySourceEntry` in config. This catches connections
|
||||
//! created before the memory_sources domain existed.
|
||||
//!
|
||||
//! Also owns the retroactive caps migration
|
||||
//! (`apply_composio_source_caps_migration`) that gives any cap-less Composio
|
||||
//! source — enabled or disabled — conservative per-toolkit caps.
|
||||
|
||||
use crate::openhuman::config::rpc as config_rpc;
|
||||
use crate::openhuman::memory_sources::registry;
|
||||
use crate::openhuman::memory_sources::types::{MemorySourceEntry, SourceKind};
|
||||
use crate::openhuman::memory_sync::composio;
|
||||
|
||||
/// Current version of the caps migration. Bump when the migration logic changes
|
||||
/// so installs that ran an earlier revision re-run it exactly once.
|
||||
const CURRENT_CAPS_MIGRATION_VERSION: u32 = 1;
|
||||
|
||||
pub async fn ensure_composio_sources() {
|
||||
tracing::debug!("[memory_sources:reconcile] starting composio reconciliation");
|
||||
|
||||
@@ -67,6 +76,100 @@ pub async fn ensure_composio_sources() {
|
||||
"[memory_sources:reconcile] composio reconciliation complete"
|
||||
);
|
||||
}
|
||||
|
||||
// Run the one-time caps migration after the reconcile loop so any
|
||||
// sources upserted just above are also considered.
|
||||
if let Err(e) = apply_composio_source_caps_migration().await {
|
||||
tracing::warn!(
|
||||
error = %e,
|
||||
"[memory_sources:reconcile] caps migration failed (non-fatal, will retry next time)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply conservative default caps in-place to every cap-less source.
|
||||
///
|
||||
/// For a Composio source with no `max_items`/`sync_depth_days`, writes the
|
||||
/// per-toolkit defaults and enables it (a no-op when already enabled) — an
|
||||
/// already-enabled, cap-less source would otherwise sync at the provider's large
|
||||
/// internal ceiling instead of the cheap default. For other kinds, fills any unset
|
||||
/// kind-specific caps via `apply_kind_defaults`. User-customised caps (non-None)
|
||||
/// are never overwritten. Returns the number of Composio entries that received
|
||||
/// defaults. Pure (no I/O) so it can be unit-tested directly.
|
||||
fn apply_caps_defaults_to_entries(sources: &mut [MemorySourceEntry]) -> u32 {
|
||||
let mut applied = 0u32;
|
||||
for source in sources.iter_mut() {
|
||||
match source.kind {
|
||||
SourceKind::Composio => {
|
||||
// Apply to enabled AND disabled cap-less sources; skip entries the
|
||||
// user has already customised (any non-None cap).
|
||||
if source.max_items.is_none() && source.sync_depth_days.is_none() {
|
||||
let toolkit = source.toolkit.as_deref().unwrap_or("");
|
||||
let (max_items, sync_depth_days) =
|
||||
registry::memory_sync_defaults_for_toolkit(toolkit);
|
||||
tracing::debug!(
|
||||
id = %source.id,
|
||||
toolkit = %toolkit,
|
||||
was_enabled = source.enabled,
|
||||
max_items = ?max_items,
|
||||
sync_depth_days = ?sync_depth_days,
|
||||
"[memory_sources:reconcile] caps migration: applying conservative defaults"
|
||||
);
|
||||
source.enabled = true;
|
||||
source.max_items = max_items;
|
||||
source.sync_depth_days = sync_depth_days;
|
||||
applied += 1;
|
||||
}
|
||||
}
|
||||
// Apply non-composio kind defaults for entries with all-None caps.
|
||||
_ => {
|
||||
// Use the rpc::apply_kind_defaults helper so the same
|
||||
// conservative values are applied consistently.
|
||||
crate::openhuman::memory_sources::rpc::apply_kind_defaults(source);
|
||||
}
|
||||
}
|
||||
}
|
||||
applied
|
||||
}
|
||||
|
||||
/// Retroactive migration: give any cap-less Composio source — enabled or
|
||||
/// disabled — conservative per-toolkit caps so its first sync stays cheap.
|
||||
///
|
||||
/// Version-gated by `Config.composio_source_caps_migration_version`: runs once per
|
||||
/// `CURRENT_CAPS_MIGRATION_VERSION` bump (installs that ran an earlier revision
|
||||
/// re-run it exactly once). Entries the user has already customised (non-None caps)
|
||||
/// are left untouched.
|
||||
pub async fn apply_composio_source_caps_migration() -> Result<(), String> {
|
||||
let mut config = config_rpc::load_config_with_timeout().await?;
|
||||
|
||||
if config.composio_source_caps_migration_version >= CURRENT_CAPS_MIGRATION_VERSION {
|
||||
tracing::debug!(
|
||||
version = config.composio_source_caps_migration_version,
|
||||
"[memory_sources:reconcile] caps migration already at current version; skipping"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
from_version = config.composio_source_caps_migration_version,
|
||||
to_version = CURRENT_CAPS_MIGRATION_VERSION,
|
||||
"[memory_sources:reconcile] applying composio source caps migration"
|
||||
);
|
||||
|
||||
let migrated_count = apply_caps_defaults_to_entries(&mut config.memory_sources);
|
||||
|
||||
config.composio_source_caps_migration_version = CURRENT_CAPS_MIGRATION_VERSION;
|
||||
config
|
||||
.save()
|
||||
.await
|
||||
.map_err(|e| format!("caps migration: failed to save config: {e:#}"))?;
|
||||
|
||||
tracing::info!(
|
||||
migrated = migrated_count,
|
||||
"[memory_sources:reconcile] caps migration complete"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn title_case(s: &str) -> String {
|
||||
@@ -92,6 +195,109 @@ fn short_id(id: &str) -> &str {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::openhuman::memory_sources::types::{MemorySourceEntry, SourceKind};
|
||||
|
||||
fn make_composio_entry(
|
||||
id: &str,
|
||||
toolkit: &str,
|
||||
enabled: bool,
|
||||
max_items: Option<u32>,
|
||||
sync_depth_days: Option<u32>,
|
||||
) -> MemorySourceEntry {
|
||||
MemorySourceEntry {
|
||||
id: id.to_string(),
|
||||
kind: SourceKind::Composio,
|
||||
label: toolkit.to_string(),
|
||||
enabled,
|
||||
toolkit: Some(toolkit.to_string()),
|
||||
connection_id: Some(format!("conn_{id}")),
|
||||
path: None,
|
||||
glob: None,
|
||||
url: None,
|
||||
branch: None,
|
||||
paths: Vec::new(),
|
||||
max_commits: None,
|
||||
max_issues: None,
|
||||
max_prs: None,
|
||||
query: None,
|
||||
since_days: None,
|
||||
max_items,
|
||||
selector: None,
|
||||
max_tokens_per_sync: None,
|
||||
max_cost_per_sync_usd: None,
|
||||
sync_depth_days,
|
||||
}
|
||||
}
|
||||
|
||||
/// Exercises the real migration transform (`apply_caps_defaults_to_entries`)
|
||||
/// so the tests cannot drift from the production predicate.
|
||||
fn run_migration_on_entries(sources: &mut Vec<MemorySourceEntry>) -> u32 {
|
||||
apply_caps_defaults_to_entries(sources)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn migration_flips_disabled_capless_entry_to_enabled_with_caps() {
|
||||
let mut sources = vec![make_composio_entry("s1", "gmail", false, None, None)];
|
||||
let count = run_migration_on_entries(&mut sources);
|
||||
assert_eq!(count, 1);
|
||||
assert!(sources[0].enabled);
|
||||
assert_eq!(sources[0].max_items, Some(100));
|
||||
assert_eq!(sources[0].sync_depth_days, Some(30));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn migration_applies_defaults_to_enabled_capless_entry() {
|
||||
// An already-enabled but cap-less source must also receive defaults —
|
||||
// otherwise its first sync runs at the provider's large internal ceiling.
|
||||
let mut sources = vec![make_composio_entry("s2", "slack", true, None, None)];
|
||||
let count = run_migration_on_entries(&mut sources);
|
||||
assert_eq!(count, 1);
|
||||
assert!(sources[0].enabled);
|
||||
assert_eq!(sources[0].max_items, Some(50));
|
||||
assert_eq!(sources[0].sync_depth_days, Some(14));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn migration_leaves_user_customised_caps_untouched() {
|
||||
// User set max_items explicitly → migration should not override.
|
||||
let mut sources = vec![make_composio_entry("s3", "notion", false, Some(5), None)];
|
||||
let count = run_migration_on_entries(&mut sources);
|
||||
assert_eq!(count, 0, "entry with user-set caps must not be migrated");
|
||||
assert!(!sources[0].enabled, "enabled must not be flipped");
|
||||
assert_eq!(sources[0].max_items, Some(5), "user cap must be preserved");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn migration_is_noop_on_empty_list() {
|
||||
let mut sources: Vec<MemorySourceEntry> = vec![];
|
||||
let count = run_migration_on_entries(&mut sources);
|
||||
assert_eq!(count, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn migration_applies_correct_defaults_per_toolkit() {
|
||||
let toolkits = [
|
||||
("gmail", Some(100u32), Some(30u32)),
|
||||
("slack", Some(50), Some(14)),
|
||||
("notion", Some(30), Some(30)),
|
||||
("linear", Some(50), Some(30)),
|
||||
("clickup", Some(50), Some(30)),
|
||||
("github", Some(50), Some(30)),
|
||||
("unknown", Some(30), Some(14)),
|
||||
];
|
||||
for (toolkit, exp_items, exp_days) in &toolkits {
|
||||
let mut sources = vec![make_composio_entry("sid", toolkit, false, None, None)];
|
||||
run_migration_on_entries(&mut sources);
|
||||
assert_eq!(
|
||||
sources[0].max_items, *exp_items,
|
||||
"max_items mismatch for toolkit={toolkit}"
|
||||
);
|
||||
assert_eq!(
|
||||
sources[0].sync_depth_days, *exp_days,
|
||||
"sync_depth_days mismatch for toolkit={toolkit}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn short_id_truncates_ascii() {
|
||||
|
||||
@@ -7,6 +7,27 @@
|
||||
use crate::openhuman::config::rpc as config_rpc;
|
||||
use crate::openhuman::memory_sources::types::{MemorySourceEntry, SourceKind};
|
||||
|
||||
/// Conservative default sync caps for a Composio toolkit, keyed by toolkit slug.
|
||||
///
|
||||
/// Single source of truth for the cheap out-of-the-box sync volume. Applied to a
|
||||
/// source entry when it is first registered (`upsert_composio_source`, insert-only)
|
||||
/// and by the one-time caps migration (`reconcile::apply_composio_source_caps_migration`)
|
||||
/// for cap-less entries. Never overwrites a user-customised cap.
|
||||
///
|
||||
/// Returns `(max_items, sync_depth_days)`.
|
||||
pub fn memory_sync_defaults_for_toolkit(toolkit: &str) -> (Option<u32>, Option<u32>) {
|
||||
match toolkit {
|
||||
"gmail" => (Some(100), Some(30)),
|
||||
"slack" => (Some(50), Some(14)),
|
||||
"notion" => (Some(30), Some(30)),
|
||||
"linear" => (Some(50), Some(30)),
|
||||
"clickup" => (Some(50), Some(30)),
|
||||
"github" => (Some(50), Some(30)),
|
||||
// Generic fallback for any toolkit not listed above.
|
||||
_ => (Some(30), Some(14)),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn list_sources() -> Result<Vec<MemorySourceEntry>, String> {
|
||||
let config = config_rpc::load_config_with_timeout().await?;
|
||||
Ok(config.memory_sources.clone())
|
||||
@@ -38,7 +59,6 @@ pub async fn add_source(entry: MemorySourceEntry) -> Result<MemorySourceEntry, S
|
||||
tracing::info!(
|
||||
id = %entry.id,
|
||||
kind = %entry.kind.as_str(),
|
||||
label = %entry.label,
|
||||
"[memory_sources] adding source"
|
||||
);
|
||||
|
||||
@@ -111,6 +131,15 @@ pub async fn update_source(
|
||||
if let Some(v) = patch.sync_depth_days {
|
||||
entry.sync_depth_days = Some(v);
|
||||
}
|
||||
if let Some(v) = patch.max_commits {
|
||||
entry.max_commits = Some(v);
|
||||
}
|
||||
if let Some(v) = patch.max_issues {
|
||||
entry.max_issues = Some(v);
|
||||
}
|
||||
if let Some(v) = patch.max_prs {
|
||||
entry.max_prs = Some(v);
|
||||
}
|
||||
|
||||
entry.validate()?;
|
||||
let updated = entry.clone();
|
||||
@@ -202,11 +231,19 @@ pub async fn upsert_composio_source(
|
||||
return Ok(updated);
|
||||
}
|
||||
|
||||
let (default_max_items, default_sync_depth_days) = memory_sync_defaults_for_toolkit(toolkit);
|
||||
tracing::debug!(
|
||||
toolkit = %toolkit,
|
||||
max_items = ?default_max_items,
|
||||
sync_depth_days = ?default_sync_depth_days,
|
||||
"[memory_sources] applying conservative defaults for new composio source"
|
||||
);
|
||||
|
||||
let entry = MemorySourceEntry {
|
||||
id: format!("src_{}", uuid::Uuid::new_v4().as_simple()),
|
||||
kind: SourceKind::Composio,
|
||||
label: label.to_string(),
|
||||
enabled: false,
|
||||
enabled: true,
|
||||
toolkit: Some(toolkit.to_string()),
|
||||
connection_id: Some(connection_id.to_string()),
|
||||
path: None,
|
||||
@@ -219,11 +256,11 @@ pub async fn upsert_composio_source(
|
||||
max_prs: None,
|
||||
query: None,
|
||||
since_days: None,
|
||||
max_items: None,
|
||||
max_items: default_max_items,
|
||||
selector: None,
|
||||
max_tokens_per_sync: None,
|
||||
max_cost_per_sync_usd: None,
|
||||
sync_depth_days: None,
|
||||
sync_depth_days: default_sync_depth_days,
|
||||
};
|
||||
config.memory_sources.push(entry.clone());
|
||||
config
|
||||
@@ -275,12 +312,106 @@ pub struct MemorySourcePatch {
|
||||
pub max_cost_per_sync_usd: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub sync_depth_days: Option<u32>,
|
||||
// ── GithubRepo-specific caps (previously missing from patch) ──
|
||||
#[serde(default)]
|
||||
pub max_commits: Option<u32>,
|
||||
#[serde(default)]
|
||||
pub max_issues: Option<u32>,
|
||||
#[serde(default)]
|
||||
pub max_prs: Option<u32>,
|
||||
}
|
||||
|
||||
/// Enable ALL configured memory sources and clear every per-source cap,
|
||||
/// giving the user unrestricted access ("All In" mode).
|
||||
///
|
||||
/// For each source in `config.memory_sources`:
|
||||
/// - Sets `enabled = true`.
|
||||
/// - Clears `max_items`, `since_days`, `sync_depth_days`,
|
||||
/// `max_commits`, `max_issues`, `max_prs`,
|
||||
/// `max_tokens_per_sync`, `max_cost_per_sync_usd` to `None`.
|
||||
///
|
||||
/// Saves config once after all mutations and returns the updated entries.
|
||||
pub async fn apply_all_in() -> Result<Vec<MemorySourceEntry>, String> {
|
||||
let mut config = config_rpc::load_config_with_timeout().await?;
|
||||
|
||||
tracing::info!(
|
||||
count = config.memory_sources.len(),
|
||||
"[memory_sources] apply_all_in: enabling all sources and clearing caps"
|
||||
);
|
||||
|
||||
for source in &mut config.memory_sources {
|
||||
tracing::debug!(
|
||||
id = %source.id,
|
||||
kind = %source.kind.as_str(),
|
||||
"[memory_sources] apply_all_in: enabling source and clearing caps"
|
||||
);
|
||||
source.enabled = true;
|
||||
source.max_items = None;
|
||||
source.since_days = None;
|
||||
source.sync_depth_days = None;
|
||||
source.max_commits = None;
|
||||
source.max_issues = None;
|
||||
source.max_prs = None;
|
||||
source.max_tokens_per_sync = None;
|
||||
source.max_cost_per_sync_usd = None;
|
||||
}
|
||||
|
||||
let updated = config.memory_sources.clone();
|
||||
|
||||
config
|
||||
.save()
|
||||
.await
|
||||
.map_err(|e| format!("apply_all_in: failed to save config: {e:#}"))?;
|
||||
|
||||
tracing::info!(
|
||||
count = updated.len(),
|
||||
"[memory_sources] apply_all_in: complete — all sources enabled, all caps cleared"
|
||||
);
|
||||
|
||||
Ok(updated)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn composio_defaults_for_known_toolkits() {
|
||||
assert_eq!(
|
||||
memory_sync_defaults_for_toolkit("gmail"),
|
||||
(Some(100), Some(30))
|
||||
);
|
||||
assert_eq!(
|
||||
memory_sync_defaults_for_toolkit("slack"),
|
||||
(Some(50), Some(14))
|
||||
);
|
||||
assert_eq!(
|
||||
memory_sync_defaults_for_toolkit("notion"),
|
||||
(Some(30), Some(30))
|
||||
);
|
||||
assert_eq!(
|
||||
memory_sync_defaults_for_toolkit("linear"),
|
||||
(Some(50), Some(30))
|
||||
);
|
||||
assert_eq!(
|
||||
memory_sync_defaults_for_toolkit("clickup"),
|
||||
(Some(50), Some(30))
|
||||
);
|
||||
assert_eq!(
|
||||
memory_sync_defaults_for_toolkit("github"),
|
||||
(Some(50), Some(30))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn composio_defaults_for_generic_fallback() {
|
||||
assert_eq!(
|
||||
memory_sync_defaults_for_toolkit("unknown_toolkit_xyz"),
|
||||
(Some(30), Some(14))
|
||||
);
|
||||
assert_eq!(memory_sync_defaults_for_toolkit(""), (Some(30), Some(14)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn memory_source_patch_deserializes_partial() {
|
||||
let json = serde_json::json!({ "label": "New label", "enabled": false });
|
||||
@@ -289,4 +420,29 @@ mod tests {
|
||||
assert_eq!(patch.enabled, Some(false));
|
||||
assert!(patch.toolkit.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn memory_source_patch_round_trips_github_limit_fields() {
|
||||
let json = serde_json::json!({
|
||||
"max_commits": 100,
|
||||
"max_issues": 50,
|
||||
"max_prs": 25
|
||||
});
|
||||
let patch: MemorySourcePatch = serde_json::from_value(json).unwrap();
|
||||
assert_eq!(patch.max_commits, Some(100));
|
||||
assert_eq!(patch.max_issues, Some(50));
|
||||
assert_eq!(patch.max_prs, Some(25));
|
||||
// Unset fields must be None (serde(default))
|
||||
assert!(patch.label.is_none());
|
||||
assert!(patch.enabled.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn memory_source_patch_defaults_github_fields_to_none() {
|
||||
let json = serde_json::json!({ "enabled": true });
|
||||
let patch: MemorySourcePatch = serde_json::from_value(json).unwrap();
|
||||
assert!(patch.max_commits.is_none());
|
||||
assert!(patch.max_issues.is_none());
|
||||
assert!(patch.max_prs.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -105,7 +105,7 @@ pub async fn add_rpc(req: AddRequest) -> Result<RpcOutcome<AddResponse>, String>
|
||||
"[memory_sources] add_rpc: entry"
|
||||
);
|
||||
|
||||
let entry = MemorySourceEntry {
|
||||
let mut entry = MemorySourceEntry {
|
||||
id: format!("src_{}", uuid::Uuid::new_v4().as_simple()),
|
||||
kind: req.kind,
|
||||
label: req.label,
|
||||
@@ -117,9 +117,6 @@ pub async fn add_rpc(req: AddRequest) -> Result<RpcOutcome<AddResponse>, String>
|
||||
url: req.url,
|
||||
branch: req.branch,
|
||||
paths: req.paths,
|
||||
// Per-type GitHub sync caps default to DEFAULT_GITHUB_ITEM_LIMIT
|
||||
// (2000) in the reader when None. Overrides are applied via the
|
||||
// update/patch path (`MemorySourcePatch`).
|
||||
max_commits: req.max_commits,
|
||||
max_issues: req.max_issues,
|
||||
max_prs: req.max_prs,
|
||||
@@ -132,10 +129,48 @@ pub async fn add_rpc(req: AddRequest) -> Result<RpcOutcome<AddResponse>, String>
|
||||
sync_depth_days: req.sync_depth_days,
|
||||
};
|
||||
|
||||
// Apply conservative per-kind defaults when the caller left caps unset.
|
||||
apply_kind_defaults(&mut entry);
|
||||
|
||||
let source = registry::add_source(entry).await?;
|
||||
Ok(RpcOutcome::new(AddResponse { source }, vec![]))
|
||||
}
|
||||
|
||||
/// Apply conservative per-kind cap defaults to a new source entry.
|
||||
///
|
||||
/// Only fills fields that are still `None` — never overwrites a
|
||||
/// caller-supplied value. This mirrors the retroactive migration logic in
|
||||
/// `reconcile::apply_composio_source_caps_migration` so the same defaults
|
||||
/// are applied consistently at creation time and during migration.
|
||||
pub fn apply_kind_defaults(entry: &mut MemorySourceEntry) {
|
||||
match entry.kind {
|
||||
SourceKind::GithubRepo => {
|
||||
if entry.max_prs.is_none() {
|
||||
entry.max_prs = Some(10);
|
||||
}
|
||||
if entry.max_issues.is_none() {
|
||||
entry.max_issues = Some(10);
|
||||
}
|
||||
if entry.max_commits.is_none() {
|
||||
entry.max_commits = Some(50);
|
||||
}
|
||||
}
|
||||
SourceKind::RssFeed => {
|
||||
if entry.max_items.is_none() {
|
||||
entry.max_items = Some(20);
|
||||
}
|
||||
}
|
||||
SourceKind::TwitterQuery => {
|
||||
if entry.since_days.is_none() {
|
||||
entry.since_days = Some(7);
|
||||
}
|
||||
}
|
||||
// Folder / WebPage / Composio: no defaults to apply here.
|
||||
// Composio defaults are set at upsert time in registry::upsert_composio_source.
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Update ──
|
||||
|
||||
#[derive(Debug, serde::Deserialize)]
|
||||
@@ -394,3 +429,72 @@ pub async fn monthly_cost_summary_rpc() -> Result<RpcOutcome<MonthlyCostSummaryR
|
||||
vec![],
|
||||
))
|
||||
}
|
||||
|
||||
// ── Apply All In ──
|
||||
|
||||
/// Response returned by `memory_sources_apply_all_in`.
|
||||
#[derive(Debug, serde::Serialize)]
|
||||
pub struct AllInResponse {
|
||||
/// All memory source entries after the "all in" transformation
|
||||
/// (every source enabled, every cap cleared).
|
||||
pub sources: Vec<MemorySourceEntry>,
|
||||
/// Number of sync tasks spawned (one per enabled source).
|
||||
pub sync_triggered: u32,
|
||||
}
|
||||
|
||||
/// Enable ALL memory sources, clear all caps, and trigger a sync for
|
||||
/// every source.
|
||||
///
|
||||
/// Returns immediately with the updated source list and the number of
|
||||
/// syncs queued. Individual syncs run in the background and publish
|
||||
/// `MemorySyncStageChanged` events as they progress.
|
||||
pub async fn apply_all_in_rpc() -> Result<RpcOutcome<AllInResponse>, String> {
|
||||
tracing::info!("[memory_sources] apply_all_in_rpc: entry");
|
||||
|
||||
// Enable all sources and clear caps.
|
||||
let sources = registry::apply_all_in().await?;
|
||||
|
||||
// Trigger a background sync for every enabled source.
|
||||
let config = config_rpc::load_config_with_timeout().await?;
|
||||
let mut sync_triggered: u32 = 0;
|
||||
|
||||
for source in &sources {
|
||||
if !source.enabled {
|
||||
continue;
|
||||
}
|
||||
tracing::debug!(
|
||||
source_id = %source.id,
|
||||
kind = %source.kind.as_str(),
|
||||
"[memory_sources] apply_all_in_rpc: triggering sync"
|
||||
);
|
||||
match crate::openhuman::memory_sources::sync::sync_source(source.clone(), config.clone())
|
||||
.await
|
||||
{
|
||||
Ok(()) => {
|
||||
sync_triggered += 1;
|
||||
}
|
||||
Err(e) => {
|
||||
// Non-fatal: log and continue — best-effort sync trigger.
|
||||
tracing::warn!(
|
||||
source_id = %source.id,
|
||||
error = %e,
|
||||
"[memory_sources] apply_all_in_rpc: sync trigger failed for source"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
sources = sources.len(),
|
||||
sync_triggered,
|
||||
"[memory_sources] apply_all_in_rpc: complete"
|
||||
);
|
||||
|
||||
Ok(RpcOutcome::new(
|
||||
AllInResponse {
|
||||
sources,
|
||||
sync_triggered,
|
||||
},
|
||||
vec![],
|
||||
))
|
||||
}
|
||||
|
||||
@@ -55,6 +55,24 @@ fn kind_specific_fields() -> Vec<FieldSchema> {
|
||||
comment: "Path filters for github_repo sources.",
|
||||
required: false,
|
||||
},
|
||||
FieldSchema {
|
||||
name: "max_commits",
|
||||
ty: TypeSchema::Option(Box::new(TypeSchema::U64)),
|
||||
comment: "Max commits per sync for github_repo sources.",
|
||||
required: false,
|
||||
},
|
||||
FieldSchema {
|
||||
name: "max_issues",
|
||||
ty: TypeSchema::Option(Box::new(TypeSchema::U64)),
|
||||
comment: "Max issues per sync for github_repo sources.",
|
||||
required: false,
|
||||
},
|
||||
FieldSchema {
|
||||
name: "max_prs",
|
||||
ty: TypeSchema::Option(Box::new(TypeSchema::U64)),
|
||||
comment: "Max pull requests per sync for github_repo sources.",
|
||||
required: false,
|
||||
},
|
||||
FieldSchema {
|
||||
name: "query",
|
||||
ty: TypeSchema::Option(Box::new(TypeSchema::String)),
|
||||
@@ -70,7 +88,7 @@ fn kind_specific_fields() -> Vec<FieldSchema> {
|
||||
FieldSchema {
|
||||
name: "max_items",
|
||||
ty: TypeSchema::Option(Box::new(TypeSchema::U64)),
|
||||
comment: "Maximum items for rss_feed sources.",
|
||||
comment: "Maximum items for rss_feed or composio sources.",
|
||||
required: false,
|
||||
},
|
||||
FieldSchema {
|
||||
@@ -114,6 +132,7 @@ pub fn all_controller_schemas() -> Vec<ControllerSchema> {
|
||||
schemas("sync_audit_log"),
|
||||
schemas("estimate_sync_cost"),
|
||||
schemas("monthly_cost_summary"),
|
||||
schemas("apply_all_in"),
|
||||
]
|
||||
}
|
||||
|
||||
@@ -167,6 +186,10 @@ pub fn all_registered_controllers() -> Vec<RegisteredController> {
|
||||
schema: schemas("monthly_cost_summary"),
|
||||
handler: handle_monthly_cost_summary,
|
||||
},
|
||||
RegisteredController {
|
||||
schema: schemas("apply_all_in"),
|
||||
handler: handle_apply_all_in,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
@@ -487,6 +510,29 @@ pub fn schemas(function: &str) -> ControllerSchema {
|
||||
},
|
||||
],
|
||||
},
|
||||
"apply_all_in" => ControllerSchema {
|
||||
namespace: NAMESPACE,
|
||||
function: "apply_all_in",
|
||||
description: "Enable ALL memory sources, clear all per-source caps, \
|
||||
and trigger a background sync for every source. \
|
||||
Returns immediately with the updated source list and \
|
||||
the count of sync tasks queued.",
|
||||
inputs: vec![],
|
||||
outputs: vec![
|
||||
FieldSchema {
|
||||
name: "sources",
|
||||
ty: TypeSchema::Array(Box::new(TypeSchema::Ref("MemorySourceEntry"))),
|
||||
comment: "All memory sources after the all-in transformation.",
|
||||
required: true,
|
||||
},
|
||||
FieldSchema {
|
||||
name: "sync_triggered",
|
||||
ty: TypeSchema::U64,
|
||||
comment: "Number of sync tasks spawned.",
|
||||
required: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
other => panic!("unknown memory_sources schema function: {other}"),
|
||||
}
|
||||
}
|
||||
@@ -563,6 +609,10 @@ fn handle_monthly_cost_summary(_params: Map<String, Value>) -> ControllerFuture
|
||||
Box::pin(async move { to_json(rpc::monthly_cost_summary_rpc().await?) })
|
||||
}
|
||||
|
||||
fn handle_apply_all_in(_params: Map<String, Value>) -> ControllerFuture {
|
||||
Box::pin(async move { to_json(rpc::apply_all_in_rpc().await?) })
|
||||
}
|
||||
|
||||
fn parse_value<T: DeserializeOwned>(v: Value) -> Result<T, String> {
|
||||
serde_json::from_value(v).map_err(|e| format!("invalid params: {e}"))
|
||||
}
|
||||
|
||||
@@ -470,7 +470,33 @@ impl EventHandler for ComposioConnectionCreatedSubscriber {
|
||||
return;
|
||||
}
|
||||
};
|
||||
let Some(ctx) = ProviderContext::from_config(
|
||||
// Look up per-source caps from the memory_sources registry.
|
||||
// Non-fatal: if the lookup fails we proceed without caps.
|
||||
//
|
||||
// upsert_composio_source runs AFTER this block (below), so for
|
||||
// brand-new connections the entry may not exist yet. In that case
|
||||
// fall back to the per-toolkit defaults so the first sync is still
|
||||
// capped. list_enabled_by_kind would also drop disabled-but-
|
||||
// configured entries, so we use list_sources() and filter ourselves.
|
||||
let (src_max_items, src_sync_depth_days) = {
|
||||
let registry_sources = crate::openhuman::memory_sources::list_sources()
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
registry_sources
|
||||
.iter()
|
||||
.find(|s| {
|
||||
s.kind == crate::openhuman::memory_sources::SourceKind::Composio
|
||||
&& s.connection_id.as_deref() == Some(connection_id.as_str())
|
||||
})
|
||||
.map(|s| (s.max_items, s.sync_depth_days))
|
||||
.unwrap_or_else(|| {
|
||||
crate::openhuman::memory_sources::memory_sync_defaults_for_toolkit(
|
||||
toolkit.as_str(),
|
||||
)
|
||||
})
|
||||
};
|
||||
|
||||
let Some(mut ctx) = ProviderContext::from_config(
|
||||
Arc::new(config),
|
||||
toolkit.clone(),
|
||||
Some(connection_id.clone()),
|
||||
@@ -482,6 +508,17 @@ impl EventHandler for ComposioConnectionCreatedSubscriber {
|
||||
return;
|
||||
};
|
||||
|
||||
ctx.max_items = src_max_items;
|
||||
ctx.sync_depth_days = src_sync_depth_days;
|
||||
|
||||
tracing::debug!(
|
||||
toolkit = %toolkit,
|
||||
connection_id = %connection_id,
|
||||
max_items = ?src_max_items,
|
||||
sync_depth_days = ?src_sync_depth_days,
|
||||
"[composio:bus] caps from registry for connection_created"
|
||||
);
|
||||
|
||||
// `wait_for_connection_active` is a backend-only metadata
|
||||
// probe (`list_connections`). Resolve a backend
|
||||
// `ComposioClient` from the live config for it; direct-mode
|
||||
|
||||
@@ -151,11 +151,35 @@ pub async fn run_connection_sync(
|
||||
))
|
||||
})?;
|
||||
|
||||
// Look up the source entry to obtain any user-configured caps.
|
||||
// Non-fatal: if the registry read fails we proceed uncapped.
|
||||
let (src_max_items, src_sync_depth_days) = {
|
||||
let registry_sources = crate::openhuman::memory_sources::list_enabled_by_kind(
|
||||
crate::openhuman::memory_sources::SourceKind::Composio,
|
||||
)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
registry_sources
|
||||
.iter()
|
||||
.find(|s| s.connection_id.as_deref() == Some(&target.connection_id))
|
||||
.map(|s| (s.max_items, s.sync_depth_days))
|
||||
.unwrap_or((None, None))
|
||||
};
|
||||
|
||||
tracing::debug!(
|
||||
connection_id = %target.connection_id,
|
||||
max_items = ?src_max_items,
|
||||
sync_depth_days = ?src_sync_depth_days,
|
||||
"[composio:sync] run_connection_sync: caps from registry"
|
||||
);
|
||||
|
||||
let ctx = ProviderContext {
|
||||
config: std::sync::Arc::new(config),
|
||||
toolkit: target.toolkit,
|
||||
connection_id: Some(target.connection_id),
|
||||
usage: Default::default(),
|
||||
max_items: src_max_items,
|
||||
sync_depth_days: src_sync_depth_days,
|
||||
};
|
||||
|
||||
let sync_result = provider.sync(&ctx, reason).await;
|
||||
|
||||
@@ -313,6 +313,29 @@ pub(crate) async fn run_one_tick() -> Result<(), String> {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Look up per-source caps from the memory_sources registry.
|
||||
// Non-fatal: if the lookup fails we proceed without caps.
|
||||
let (src_max_items, src_sync_depth_days) = {
|
||||
let registry_sources = crate::openhuman::memory_sources::list_enabled_by_kind(
|
||||
crate::openhuman::memory_sources::SourceKind::Composio,
|
||||
)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
registry_sources
|
||||
.iter()
|
||||
.find(|s| s.connection_id.as_deref() == Some(conn.id.as_str()))
|
||||
.map(|s| (s.max_items, s.sync_depth_days))
|
||||
.unwrap_or((None, None))
|
||||
};
|
||||
|
||||
tracing::debug!(
|
||||
toolkit = %toolkit,
|
||||
connection_id = %conn.id,
|
||||
max_items = ?src_max_items,
|
||||
sync_depth_days = ?src_sync_depth_days,
|
||||
"[composio:periodic] caps from registry"
|
||||
);
|
||||
|
||||
// Build a context tied to this specific connection and dispatch.
|
||||
// `ProviderContext` no longer caches a pre-baked
|
||||
// `ComposioClient` — provider methods resolve a fresh handle per
|
||||
@@ -323,6 +346,8 @@ pub(crate) async fn run_one_tick() -> Result<(), String> {
|
||||
toolkit: toolkit.clone(),
|
||||
connection_id: Some(conn.id.clone()),
|
||||
usage: Default::default(),
|
||||
max_items: src_max_items,
|
||||
sync_depth_days: src_sync_depth_days,
|
||||
};
|
||||
|
||||
tracing::debug!(
|
||||
|
||||
@@ -261,12 +261,42 @@ impl ComposioProvider for ClickUpProvider {
|
||||
_ => PAGE_SIZE,
|
||||
};
|
||||
|
||||
// ctx.max_items: route through ItemCap — page ceiling, mid-page
|
||||
// per-item break, and post-page hard stop all share one source of truth.
|
||||
let mut cap = super::super::helpers::ItemCap::new(ctx.max_items);
|
||||
let effective_max_pages = cap.max_pages(page_size, MAX_PAGES_PER_WORKSPACE);
|
||||
if ctx.max_items.is_some() && effective_max_pages < MAX_PAGES_PER_WORKSPACE {
|
||||
tracing::debug!(
|
||||
connection_id = %connection_id,
|
||||
max_items = ?ctx.max_items,
|
||||
effective_max_pages,
|
||||
"[composio:clickup] [memory_sync] applying max_items page cap"
|
||||
);
|
||||
}
|
||||
|
||||
// ctx.sync_depth_days: oldest allowed date_updated for client-side skip.
|
||||
// ClickUp's `date_updated` field is a millisecond-epoch string, so the
|
||||
// floor must also be epoch-millis (not RFC3339) for the lexicographic
|
||||
// compare in `select_pending` to work correctly.
|
||||
let oldest_allowed_time: Option<String> = ctx.sync_depth_days.map(|days| {
|
||||
let floor = chrono::Utc::now() - chrono::Duration::days(days as i64);
|
||||
let s = floor.timestamp_millis().to_string();
|
||||
tracing::debug!(
|
||||
connection_id = %connection_id,
|
||||
sync_depth_days = days,
|
||||
oldest_allowed_ms = %s,
|
||||
"[composio:clickup] [memory_sync] applying sync_depth_days floor"
|
||||
);
|
||||
s
|
||||
});
|
||||
|
||||
let mut total_fetched: usize = 0;
|
||||
let mut total_persisted: usize = 0;
|
||||
let mut newest_updated: Option<String> = None;
|
||||
let mut hit_cap_boundary = false;
|
||||
|
||||
'workspaces: for workspace_id in &workspaces {
|
||||
for page_num in 0..MAX_PAGES_PER_WORKSPACE {
|
||||
for page_num in 0..effective_max_pages {
|
||||
if state.budget_exhausted() {
|
||||
tracing::info!(
|
||||
workspace_id = %workspace_id,
|
||||
@@ -327,9 +357,27 @@ impl ComposioProvider for ClickUpProvider {
|
||||
}
|
||||
|
||||
// ── Per-item dedup + bounded-concurrency ingest ─────
|
||||
let (pending, hit_cursor_boundary) =
|
||||
let (mut pending, mut hit_cursor_boundary) =
|
||||
select_pending(&tasks, &state, &mut newest_updated);
|
||||
|
||||
// ctx.sync_depth_days: drop items updated before the depth floor. `pending` is
|
||||
// in descending timestamp order, so truncate at the first item below the floor
|
||||
// and signal cursor-boundary so pagination stops.
|
||||
if let Some(ref floor) = oldest_allowed_time {
|
||||
if let Some(cut) = pending.iter().position(|p| {
|
||||
p.updated
|
||||
.as_deref()
|
||||
.map(|t| t < floor.as_str())
|
||||
.unwrap_or(false)
|
||||
}) {
|
||||
pending.truncate(cut);
|
||||
hit_cursor_boundary = true;
|
||||
}
|
||||
}
|
||||
|
||||
// ctx.max_items: clamp the dedup'd batch to the remaining budget before ingest.
|
||||
cap.clamp_batch(&mut pending);
|
||||
|
||||
let ingestor = MemoryTreeIngestor {
|
||||
config: ctx.config.as_ref(),
|
||||
connection_id: &connection_id,
|
||||
@@ -341,6 +389,13 @@ impl ComposioProvider for ClickUpProvider {
|
||||
state.mark_synced(key);
|
||||
}
|
||||
total_persisted += outcome.persisted;
|
||||
cap.record(outcome.persisted);
|
||||
|
||||
// ctx.max_items precise cap: once the per-source cap is hit, stop paginating.
|
||||
if cap.is_reached() {
|
||||
hit_cap_boundary = true;
|
||||
break 'workspaces;
|
||||
}
|
||||
|
||||
if hit_cursor_boundary {
|
||||
tracing::debug!(
|
||||
@@ -351,6 +406,18 @@ impl ComposioProvider for ClickUpProvider {
|
||||
break;
|
||||
}
|
||||
|
||||
// ctx.max_items hard stop.
|
||||
if cap.is_reached() {
|
||||
tracing::debug!(
|
||||
workspace_id = %workspace_id,
|
||||
page = page_num,
|
||||
total_persisted,
|
||||
"[composio:clickup] [memory_sync] max_items reached, stopping pagination"
|
||||
);
|
||||
hit_cap_boundary = true;
|
||||
break 'workspaces;
|
||||
}
|
||||
|
||||
// ClickUp's filtered-team-tasks endpoint signals the last
|
||||
// page implicitly: when fewer than `page_size` results
|
||||
// come back, there are no more pages.
|
||||
@@ -367,8 +434,17 @@ impl ComposioProvider for ClickUpProvider {
|
||||
}
|
||||
|
||||
// ── Step 6: advance cursor and save state ───────────────────
|
||||
if let Some(new_cursor) = newest_updated {
|
||||
state.advance_cursor(&new_cursor);
|
||||
// Hold the cursor on a cap-truncated pass so the next sync re-scans the unseen tail.
|
||||
if !hit_cap_boundary {
|
||||
if let Some(new_cursor) = newest_updated {
|
||||
state.advance_cursor(&new_cursor);
|
||||
}
|
||||
} else {
|
||||
tracing::warn!(
|
||||
connection_id = %connection_id,
|
||||
"[composio:clickup] holding cursor — cap-truncated pass; next sync will re-scan \
|
||||
the unseen tail"
|
||||
);
|
||||
}
|
||||
state.set_last_sync_at_ms(sync::now_ms());
|
||||
state.save(&memory).await?;
|
||||
|
||||
@@ -200,8 +200,42 @@ impl ComposioProvider for GitHubProvider {
|
||||
_ => PAGE_SIZE,
|
||||
};
|
||||
|
||||
// Build the base search query.
|
||||
let query = build_search_query(&login, state.cursor.as_deref());
|
||||
// ctx.max_items: route through ItemCap — page ceiling, mid-page
|
||||
// per-item break, and post-page hard stop all share one source of truth.
|
||||
let mut cap = super::super::helpers::ItemCap::new(ctx.max_items);
|
||||
let effective_max_pages = cap.max_pages(page_size, MAX_PAGES);
|
||||
if ctx.max_items.is_some() && effective_max_pages < MAX_PAGES {
|
||||
tracing::debug!(
|
||||
connection_id = %connection_id,
|
||||
max_items = ?ctx.max_items,
|
||||
effective_max_pages,
|
||||
"[composio:github] [memory_sync] applying max_items page cap"
|
||||
);
|
||||
}
|
||||
|
||||
// ctx.sync_depth_days: inject `updated:>{date}` on first sync when set.
|
||||
let depth_query_fragment: Option<String> = if state.cursor.is_none() {
|
||||
ctx.sync_depth_days.map(|days| {
|
||||
let floor = chrono::Utc::now() - chrono::Duration::days(days as i64);
|
||||
let s = floor.format("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
tracing::debug!(
|
||||
connection_id = %connection_id,
|
||||
sync_depth_days = days,
|
||||
floor = %s,
|
||||
"[composio:github] [memory_sync] injecting updated:> date filter on first sync"
|
||||
);
|
||||
format!("updated:>{s}")
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Build the base search query — include depth fragment when set.
|
||||
let query = build_search_query_with_depth(
|
||||
&login,
|
||||
state.cursor.as_deref(),
|
||||
depth_query_fragment.as_deref(),
|
||||
);
|
||||
|
||||
let mut total_fetched: usize = 0;
|
||||
let mut total_persisted: usize = 0;
|
||||
@@ -214,8 +248,9 @@ impl ComposioProvider for GitHubProvider {
|
||||
// until its next edit. Already-synced items are skipped cheaply via
|
||||
// `is_synced` on the re-fetch, so the cost of holding is minimal.
|
||||
let mut had_ingest_failures = false;
|
||||
let mut hit_cap_boundary = false;
|
||||
|
||||
'pages: for page_num in 1..=MAX_PAGES {
|
||||
'pages: for page_num in 1..=effective_max_pages {
|
||||
if state.budget_exhausted() {
|
||||
tracing::info!(
|
||||
page = page_num,
|
||||
@@ -335,6 +370,7 @@ impl ComposioProvider for GitHubProvider {
|
||||
Ok(_chunks_written) => {
|
||||
state.mark_synced(&sync_key);
|
||||
total_persisted += 1;
|
||||
cap.record(1);
|
||||
}
|
||||
Err(e) => {
|
||||
had_ingest_failures = true;
|
||||
@@ -345,6 +381,24 @@ impl ComposioProvider for GitHubProvider {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ctx.max_items precise cap: stop mid-page so we never persist
|
||||
// more than the cap even when a single page exceeds it.
|
||||
if cap.is_reached() {
|
||||
hit_cap_boundary = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// ctx.max_items hard stop.
|
||||
if cap.is_reached() {
|
||||
tracing::debug!(
|
||||
page = page_num,
|
||||
total_persisted,
|
||||
"[composio:github] [memory_sync] max_items reached, stopping pagination"
|
||||
);
|
||||
hit_cap_boundary = true;
|
||||
break;
|
||||
}
|
||||
|
||||
// GitHub search pages are 0-indexed in terms of total results;
|
||||
@@ -366,15 +420,18 @@ impl ComposioProvider for GitHubProvider {
|
||||
// the delete-first memory-tree pipeline (#2885). `set_last_sync_at_ms`
|
||||
// still advances — that's just a heartbeat, not a fetch-window
|
||||
// boundary, so it's safe to record that we did attempt a sync.
|
||||
if !had_ingest_failures {
|
||||
// Hold the cursor on a cap-truncated pass so the next sync re-scans the unseen tail.
|
||||
if !had_ingest_failures && !hit_cap_boundary {
|
||||
if let Some(new_cursor) = newest_updated {
|
||||
state.advance_cursor(&new_cursor);
|
||||
}
|
||||
} else {
|
||||
tracing::warn!(
|
||||
connection_id = %connection_id,
|
||||
"[composio:github] holding cursor — ingest failures this pass; next sync will \
|
||||
re-fetch the failed range"
|
||||
had_ingest_failures,
|
||||
hit_cap_boundary,
|
||||
"[composio:github] holding cursor — ingest failures or cap-truncated pass; next \
|
||||
sync will re-fetch the failed range"
|
||||
);
|
||||
}
|
||||
state.set_last_sync_at_ms(sync::now_ms());
|
||||
@@ -896,3 +953,48 @@ pub(super) fn build_search_query(login: &str, cursor: Option<&str>) -> String {
|
||||
None => format!("involves:{login}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Extended variant that optionally appends a `sync_depth_days` fragment on
|
||||
/// first sync (no cursor). The `depth_fragment` is expected to be a pre-built
|
||||
/// `"updated:>{date}"` string.
|
||||
pub(super) fn build_search_query_with_depth(
|
||||
login: &str,
|
||||
cursor: Option<&str>,
|
||||
depth_fragment: Option<&str>,
|
||||
) -> String {
|
||||
match cursor {
|
||||
Some(c) => format!("involves:{login} updated:>{c}"),
|
||||
None => match depth_fragment {
|
||||
Some(fragment) => format!("involves:{login} {fragment}"),
|
||||
None => format!("involves:{login}"),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod depth_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn build_search_query_with_depth_no_cursor_no_depth() {
|
||||
let q = build_search_query_with_depth("alice", None, None);
|
||||
assert_eq!(q, "involves:alice");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_search_query_with_depth_no_cursor_with_depth() {
|
||||
let q = build_search_query_with_depth("alice", None, Some("updated:>2024-01-01T00:00:00Z"));
|
||||
assert_eq!(q, "involves:alice updated:>2024-01-01T00:00:00Z");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_search_query_with_depth_cursor_wins_over_depth() {
|
||||
// When cursor is set, depth fragment is ignored.
|
||||
let q = build_search_query_with_depth(
|
||||
"alice",
|
||||
Some("2024-06-01T00:00:00Z"),
|
||||
Some("updated:>2024-01-01T00:00:00Z"),
|
||||
);
|
||||
assert_eq!(q, "involves:alice updated:>2024-06-01T00:00:00Z");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -258,7 +258,7 @@ impl ComposioProvider for GmailProvider {
|
||||
// legitimately want the larger ceiling — and the cap only
|
||||
// kicks in when we have a prior `last_sync_at_ms` to compare
|
||||
// against, so first-ever syncs are unaffected.
|
||||
let max_pages = match reason {
|
||||
let base_max_pages = match reason {
|
||||
SyncReason::ConnectionCreated => MAX_PAGES_PER_SYNC,
|
||||
_ => match state.last_sync_at_ms {
|
||||
Some(last_ms) if sync::now_ms().saturating_sub(last_ms) < RECENT_SYNC_WINDOW_MS => {
|
||||
@@ -274,6 +274,36 @@ impl ComposioProvider for GmailProvider {
|
||||
},
|
||||
};
|
||||
|
||||
// ctx.max_items: route through ItemCap so the page ceiling, mid-page
|
||||
// clamp, and post-page hard stop all share one source of truth.
|
||||
let mut cap = super::super::helpers::ItemCap::new(ctx.max_items);
|
||||
let max_pages = cap.max_pages(page_size, base_max_pages);
|
||||
if ctx.max_items.is_some() && max_pages < base_max_pages {
|
||||
tracing::debug!(
|
||||
connection_id = %connection_id,
|
||||
max_items = ?ctx.max_items,
|
||||
page_size,
|
||||
effective_max_pages = max_pages,
|
||||
"[composio:gmail] [memory_sync] applying max_items page cap from source config"
|
||||
);
|
||||
}
|
||||
|
||||
// ctx.sync_depth_days: on first sync (no cursor), add an after:<epoch> floor.
|
||||
let depth_floor_filter: Option<String> = if state.cursor.is_none() {
|
||||
ctx.sync_depth_days.map(|days| {
|
||||
let floor_secs = super::super::helpers::epoch_floor_from_depth(days);
|
||||
tracing::debug!(
|
||||
connection_id = %connection_id,
|
||||
sync_depth_days = days,
|
||||
floor_epoch_secs = floor_secs,
|
||||
"[composio:gmail] [memory_sync] applying sync_depth_days floor on first sync"
|
||||
);
|
||||
floor_secs.to_string()
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let mut total_fetched: usize = 0;
|
||||
let mut total_persisted: usize = 0;
|
||||
let mut total_requests: u32 = 0;
|
||||
@@ -281,6 +311,7 @@ impl ComposioProvider for GmailProvider {
|
||||
let mut newest_id: Option<String> = None;
|
||||
let mut page_token: Option<String> = None;
|
||||
let mut stop_reason: &'static str = "max_pages";
|
||||
let mut hit_cap_boundary = false;
|
||||
|
||||
for page_num in 0..max_pages {
|
||||
if state.budget_exhausted() {
|
||||
@@ -321,6 +352,9 @@ impl ComposioProvider for GmailProvider {
|
||||
"[composio:gmail] using day-level filter from cursor (epoch parse failed)"
|
||||
);
|
||||
}
|
||||
} else if let Some(ref floor) = depth_floor_filter {
|
||||
// First sync with sync_depth_days: apply the epoch floor.
|
||||
query.push_str(&format!(" after:{floor}"));
|
||||
}
|
||||
|
||||
let mut args = json!({
|
||||
@@ -453,6 +487,11 @@ impl ComposioProvider for GmailProvider {
|
||||
new_messages.push(msg.clone());
|
||||
}
|
||||
|
||||
// ctx.max_items precise cap: clamp the per-page batch before ingest
|
||||
// so a single page larger than the budget is never over-persisted.
|
||||
cap.clamp_batch(&mut new_messages);
|
||||
cap.clamp_batch(&mut pending_synced_ids);
|
||||
|
||||
// Single batched ingest into memory_tree. Chunk IDs are
|
||||
// content-hashed so re-ingest of the same message is an
|
||||
// idempotent UPSERT at the SQL layer; per-message dedup above
|
||||
@@ -483,6 +522,7 @@ impl ComposioProvider for GmailProvider {
|
||||
// persist path. n is the chunk count which we log
|
||||
// for diagnostic purposes only.
|
||||
total_persisted += new_messages.len();
|
||||
cap.record(new_messages.len());
|
||||
tracing::debug!(
|
||||
page = page_num,
|
||||
new_messages = new_messages.len(),
|
||||
@@ -512,6 +552,18 @@ impl ComposioProvider for GmailProvider {
|
||||
break;
|
||||
}
|
||||
|
||||
// ctx.max_items hard stop: break once the per-source cap is reached.
|
||||
if cap.is_reached() {
|
||||
tracing::debug!(
|
||||
page = page_num,
|
||||
total_persisted,
|
||||
"[composio:gmail] [memory_sync] max_items reached, stopping pagination"
|
||||
);
|
||||
stop_reason = "max_items";
|
||||
hit_cap_boundary = true;
|
||||
break;
|
||||
}
|
||||
|
||||
// Check for next page token.
|
||||
page_token = sync::extract_page_token(&resp.data);
|
||||
if page_token.is_none() {
|
||||
@@ -522,8 +574,17 @@ impl ComposioProvider for GmailProvider {
|
||||
}
|
||||
|
||||
// ── Step 5: advance cursor and save state ───────────────────
|
||||
if let Some(new_cursor) = newest_date {
|
||||
state.advance_cursor(&new_cursor);
|
||||
// Hold the cursor on a cap-truncated pass so the next sync re-scans the unseen tail.
|
||||
if !hit_cap_boundary {
|
||||
if let Some(new_cursor) = newest_date {
|
||||
state.advance_cursor(&new_cursor);
|
||||
}
|
||||
} else {
|
||||
tracing::warn!(
|
||||
connection_id = %connection_id,
|
||||
"[composio:gmail] holding cursor — cap-truncated pass; next sync will re-scan \
|
||||
the unseen tail"
|
||||
);
|
||||
}
|
||||
if let Some(ref freshest) = newest_id {
|
||||
state.set_last_seen_id(freshest);
|
||||
@@ -616,3 +677,7 @@ impl ComposioProvider for GmailProvider {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// Cap/date-floor math lives in the shared `super::super::helpers` module
|
||||
// (`ItemCap`, `pages_for_max_items`, `epoch_floor_from_depth`) so every provider
|
||||
// shares one implementation — see that module for the unit tests.
|
||||
|
||||
@@ -46,6 +46,178 @@ pub(crate) fn merge_extra(args: &mut serde_json::Value, extra: &serde_json::Valu
|
||||
}
|
||||
}
|
||||
|
||||
// ── Cap enforcement helpers ──────────────────────────────────────────────
|
||||
|
||||
/// Compute the number of API pages needed to cover `max_items` at `page_size`
|
||||
/// items per page, rounding up.
|
||||
///
|
||||
/// Returns `u32::MAX` when `page_size == 0` to avoid division by zero;
|
||||
/// callers should treat this as "no page cap beyond the provider's own upper
|
||||
/// bound".
|
||||
pub(crate) fn pages_for_max_items(max_items: u32, page_size: u32) -> u32 {
|
||||
if page_size == 0 {
|
||||
return u32::MAX;
|
||||
}
|
||||
// Widen to u64 before the addition to prevent overflow for large cap values.
|
||||
(((max_items as u64) + (page_size as u64) - 1) / (page_size as u64)).min(u32::MAX as u64) as u32
|
||||
}
|
||||
|
||||
/// Compute the Unix epoch timestamp (seconds) for `sync_depth_days` days ago.
|
||||
/// Used to build after-date filters (e.g. Gmail `after:<epoch>`) on first sync.
|
||||
pub(crate) fn epoch_floor_from_depth(sync_depth_days: u32) -> i64 {
|
||||
let now = chrono::Utc::now();
|
||||
let floor = now - chrono::Duration::days(sync_depth_days as i64);
|
||||
floor.timestamp()
|
||||
}
|
||||
|
||||
/// Single source of truth for the per-sync `max_items` cap.
|
||||
///
|
||||
/// Every Composio provider used to open-code three near-identical blocks — a
|
||||
/// page-count cap, a mid-page clamp, and a post-page hard stop — which is how
|
||||
/// the same off-by-a-page bug ended up in five providers and was missed in a
|
||||
/// sixth. Funnelling all of them through this one type keeps the rule in one
|
||||
/// place: construct it from `ctx.max_items`, derive the page cap, clamp each
|
||||
/// batch (or check per item), and stop once the cap is reached.
|
||||
///
|
||||
/// `None` cap means "no item limit beyond the provider's own internal page
|
||||
/// ceiling" (e.g. after the user clicks "All In").
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub(crate) struct ItemCap {
|
||||
cap: Option<usize>,
|
||||
persisted: usize,
|
||||
}
|
||||
|
||||
impl ItemCap {
|
||||
/// Build from a source's `max_items` value (`None` = uncapped).
|
||||
pub(crate) fn new(max_items: Option<u32>) -> Self {
|
||||
Self {
|
||||
cap: max_items.map(|n| n as usize),
|
||||
persisted: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// The page ceiling to actually fetch: the smaller of the provider's own
|
||||
/// `fallback` (e.g. `MAX_PAGES_PER_SYNC`) and the pages needed to cover the
|
||||
/// cap. Uncapped → `fallback` unchanged.
|
||||
pub(crate) fn max_pages(&self, page_size: u32, fallback: u32) -> u32 {
|
||||
match self.cap {
|
||||
Some(cap) => pages_for_max_items(cap as u32, page_size).min(fallback),
|
||||
None => fallback,
|
||||
}
|
||||
}
|
||||
|
||||
/// How many more items may still be persisted. `None` = unlimited.
|
||||
pub(crate) fn remaining(&self) -> Option<usize> {
|
||||
self.cap.map(|cap| cap.saturating_sub(self.persisted))
|
||||
}
|
||||
|
||||
/// True once the cap is set and reached — callers `break` their pagination.
|
||||
pub(crate) fn is_reached(&self) -> bool {
|
||||
matches!(self.remaining(), Some(0))
|
||||
}
|
||||
|
||||
/// Record `n` newly-persisted items against the budget.
|
||||
pub(crate) fn record(&mut self, n: usize) {
|
||||
self.persisted = self.persisted.saturating_add(n);
|
||||
}
|
||||
|
||||
/// Truncate a to-ingest batch down to the remaining budget, so a single
|
||||
/// page larger than the cap never over-persists. No-op when uncapped.
|
||||
pub(crate) fn clamp_batch<T>(&self, batch: &mut Vec<T>) {
|
||||
if let Some(remaining) = self.remaining() {
|
||||
if batch.len() > remaining {
|
||||
batch.truncate(remaining);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod cap_helper_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn pages_for_max_items_rounds_up() {
|
||||
assert_eq!(pages_for_max_items(100, 25), 4);
|
||||
assert_eq!(pages_for_max_items(101, 25), 5);
|
||||
assert_eq!(pages_for_max_items(1, 25), 1);
|
||||
assert_eq!(pages_for_max_items(50, 50), 1);
|
||||
assert_eq!(pages_for_max_items(51, 50), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pages_for_max_items_zero_page_size() {
|
||||
assert_eq!(pages_for_max_items(100, 0), u32::MAX);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn epoch_floor_from_depth_is_in_the_past() {
|
||||
let floor = epoch_floor_from_depth(30);
|
||||
let now = chrono::Utc::now().timestamp();
|
||||
assert!(floor < now);
|
||||
let diff_days = (now - floor) / 86400;
|
||||
assert!(
|
||||
diff_days >= 29 && diff_days <= 31,
|
||||
"expected ~30 days in past, got {diff_days}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn item_cap_uncapped_is_never_reached() {
|
||||
let mut cap = ItemCap::new(None);
|
||||
assert_eq!(cap.remaining(), None);
|
||||
assert!(!cap.is_reached());
|
||||
cap.record(1_000_000);
|
||||
assert!(!cap.is_reached());
|
||||
assert_eq!(
|
||||
cap.max_pages(25, 20),
|
||||
20,
|
||||
"uncapped keeps the provider fallback"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn item_cap_tracks_remaining_and_reached() {
|
||||
let mut cap = ItemCap::new(Some(3));
|
||||
assert_eq!(cap.remaining(), Some(3));
|
||||
assert!(!cap.is_reached());
|
||||
cap.record(2);
|
||||
assert_eq!(cap.remaining(), Some(1));
|
||||
assert!(!cap.is_reached());
|
||||
cap.record(5); // saturates, never underflows
|
||||
assert_eq!(cap.remaining(), Some(0));
|
||||
assert!(cap.is_reached());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn item_cap_max_pages_is_min_of_fallback_and_needed() {
|
||||
// cap=2, page_size=50 → 1 page needed, well under the fallback.
|
||||
assert_eq!(ItemCap::new(Some(2)).max_pages(50, 20), 1);
|
||||
// cap=1000, page_size=25 → 40 pages needed, clamped to fallback 20.
|
||||
assert_eq!(ItemCap::new(Some(1000)).max_pages(25, 20), 20);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn item_cap_clamp_batch_truncates_to_remaining() {
|
||||
let cap = ItemCap::new(Some(2));
|
||||
let mut batch = vec![1, 2, 3, 4, 5];
|
||||
cap.clamp_batch(&mut batch);
|
||||
assert_eq!(batch, vec![1, 2]);
|
||||
|
||||
// Uncapped leaves the batch untouched.
|
||||
let mut full = vec![1, 2, 3];
|
||||
ItemCap::new(None).clamp_batch(&mut full);
|
||||
assert_eq!(full, vec![1, 2, 3]);
|
||||
|
||||
// After recording progress, clamp uses the reduced budget.
|
||||
let mut cap2 = ItemCap::new(Some(5));
|
||||
cap2.record(3);
|
||||
let mut batch2 = vec![1, 2, 3, 4];
|
||||
cap2.clamp_batch(&mut batch2);
|
||||
assert_eq!(batch2, vec![1, 2], "only 2 of the 5 budget remained");
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve the first array found among `array_paths` (dotted object
|
||||
/// paths), then return the first non-empty string at one of `fields`
|
||||
/// on that array's first element. Complements [`pick_str`], which
|
||||
|
||||
@@ -198,14 +198,41 @@ impl ComposioProvider for LinearProvider {
|
||||
_ => PAGE_SIZE,
|
||||
};
|
||||
|
||||
// ctx.max_items: route through ItemCap — page ceiling, mid-page
|
||||
// per-item break, and post-page hard stop all share one source of truth.
|
||||
let mut cap = super::super::helpers::ItemCap::new(ctx.max_items);
|
||||
let effective_max_pages = cap.max_pages(page_size as u32, MAX_PAGES_PER_SYNC);
|
||||
if ctx.max_items.is_some() && effective_max_pages < MAX_PAGES_PER_SYNC {
|
||||
tracing::debug!(
|
||||
connection_id = %connection_id,
|
||||
max_items = ?ctx.max_items,
|
||||
effective_max_pages,
|
||||
"[composio:linear] [memory_sync] applying max_items page cap"
|
||||
);
|
||||
}
|
||||
|
||||
// ctx.sync_depth_days: oldest allowed updatedAt for client-side skip.
|
||||
let oldest_allowed_time: Option<String> = ctx.sync_depth_days.map(|days| {
|
||||
let floor = chrono::Utc::now() - chrono::Duration::days(days as i64);
|
||||
let s = floor.to_rfc3339();
|
||||
tracing::debug!(
|
||||
connection_id = %connection_id,
|
||||
sync_depth_days = days,
|
||||
oldest_allowed = %s,
|
||||
"[composio:linear] [memory_sync] applying sync_depth_days floor"
|
||||
);
|
||||
s
|
||||
});
|
||||
|
||||
let mut total_fetched: usize = 0;
|
||||
let mut total_persisted: usize = 0;
|
||||
let mut had_persist_failures = false;
|
||||
let mut newest_updated: Option<String> = None;
|
||||
let mut after_cursor: Option<String> = None;
|
||||
let mut hit_cursor_boundary = false;
|
||||
let mut hit_cap_boundary = false;
|
||||
|
||||
for page_num in 0..MAX_PAGES_PER_SYNC {
|
||||
for page_num in 0..effective_max_pages {
|
||||
if state.budget_exhausted() {
|
||||
tracing::info!(
|
||||
page = page_num,
|
||||
@@ -256,11 +283,30 @@ impl ComposioProvider for LinearProvider {
|
||||
}
|
||||
|
||||
// ── Per-item dedup + bounded-concurrency ingest ──────────
|
||||
let (pending, page_hit_boundary) = select_pending(&issues, &state, &mut newest_updated);
|
||||
let (mut pending, page_hit_boundary) =
|
||||
select_pending(&issues, &state, &mut newest_updated);
|
||||
if page_hit_boundary {
|
||||
hit_cursor_boundary = true;
|
||||
}
|
||||
|
||||
// ctx.sync_depth_days: drop items updated before the depth floor. `pending` is
|
||||
// in descending timestamp order, so truncate at the first item below the floor
|
||||
// and signal cursor-boundary so pagination stops.
|
||||
if let Some(ref floor) = oldest_allowed_time {
|
||||
if let Some(cut) = pending.iter().position(|p| {
|
||||
p.updated
|
||||
.as_deref()
|
||||
.map(|t| t < floor.as_str())
|
||||
.unwrap_or(false)
|
||||
}) {
|
||||
pending.truncate(cut);
|
||||
hit_cursor_boundary = true;
|
||||
}
|
||||
}
|
||||
|
||||
// ctx.max_items: clamp the dedup'd batch to the remaining budget before ingest.
|
||||
cap.clamp_batch(&mut pending);
|
||||
|
||||
let ingestor = MemoryTreeIngestor {
|
||||
config: ctx.config.as_ref(),
|
||||
connection_id: &connection_id,
|
||||
@@ -270,10 +316,17 @@ impl ComposioProvider for LinearProvider {
|
||||
state.mark_synced(key);
|
||||
}
|
||||
total_persisted += outcome.persisted;
|
||||
cap.record(outcome.persisted);
|
||||
if outcome.had_failures {
|
||||
had_persist_failures = true;
|
||||
}
|
||||
|
||||
// ctx.max_items precise cap: once the per-source cap is hit, stop paginating.
|
||||
if cap.is_reached() {
|
||||
hit_cap_boundary = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if hit_cursor_boundary {
|
||||
tracing::debug!(
|
||||
page = page_num,
|
||||
@@ -282,6 +335,17 @@ impl ComposioProvider for LinearProvider {
|
||||
break;
|
||||
}
|
||||
|
||||
// ctx.max_items hard stop.
|
||||
if cap.is_reached() {
|
||||
tracing::debug!(
|
||||
page = page_num,
|
||||
total_persisted,
|
||||
"[composio:linear] [memory_sync] max_items reached, stopping pagination"
|
||||
);
|
||||
hit_cap_boundary = true;
|
||||
break;
|
||||
}
|
||||
|
||||
// Advance to the next page using Linear's cursor-based pagination.
|
||||
match sync::extract_pagination_cursor(&resp.data) {
|
||||
Some(next_cursor) => {
|
||||
@@ -298,10 +362,17 @@ impl ComposioProvider for LinearProvider {
|
||||
}
|
||||
|
||||
// ── Step 5: advance cursor and save state ────────────────────
|
||||
// Hold the cursor on a cap-truncated pass so the next sync re-scans the unseen tail.
|
||||
if had_persist_failures {
|
||||
tracing::warn!(
|
||||
"[composio:linear] persist failures seen; keeping previous cursor for retry"
|
||||
);
|
||||
} else if hit_cap_boundary {
|
||||
tracing::warn!(
|
||||
hit_cap_boundary,
|
||||
"[composio:linear] holding cursor — cap-truncated pass; next sync will re-scan \
|
||||
the unseen tail"
|
||||
);
|
||||
} else if let Some(new_cursor) = newest_updated {
|
||||
state.advance_cursor(&new_cursor);
|
||||
}
|
||||
|
||||
@@ -186,6 +186,33 @@ impl ComposioProvider for NotionProvider {
|
||||
_ => PAGE_SIZE,
|
||||
};
|
||||
|
||||
// ctx.max_items: route through ItemCap — page ceiling, mid-page
|
||||
// per-item break, and post-page hard stop all share one source of truth.
|
||||
let mut cap = super::super::helpers::ItemCap::new(ctx.max_items);
|
||||
let effective_max_pages = cap.max_pages(page_size, MAX_PAGES_PER_SYNC);
|
||||
if ctx.max_items.is_some() && effective_max_pages < MAX_PAGES_PER_SYNC {
|
||||
tracing::debug!(
|
||||
connection_id = %connection_id,
|
||||
max_items = ?ctx.max_items,
|
||||
effective_max_pages,
|
||||
"[composio:notion] [memory_sync] applying max_items page cap"
|
||||
);
|
||||
}
|
||||
|
||||
// ctx.sync_depth_days: compute the oldest allowed edited_time string
|
||||
// for client-side skipping of stale results.
|
||||
let oldest_allowed_time: Option<String> = ctx.sync_depth_days.map(|days| {
|
||||
let floor = chrono::Utc::now() - chrono::Duration::days(days as i64);
|
||||
let s = floor.to_rfc3339();
|
||||
tracing::debug!(
|
||||
connection_id = %connection_id,
|
||||
sync_depth_days = days,
|
||||
oldest_allowed = %s,
|
||||
"[composio:notion] [memory_sync] applying sync_depth_days floor"
|
||||
);
|
||||
s
|
||||
});
|
||||
|
||||
let mut total_fetched: usize = 0;
|
||||
let mut total_persisted: usize = 0;
|
||||
let mut newest_edited_time: Option<String> = None;
|
||||
@@ -198,8 +225,9 @@ impl ComposioProvider for NotionProvider {
|
||||
// its next edit. Already-synced items are skipped cheaply via
|
||||
// `is_synced` on the re-fetch, so the cost of holding is minimal.
|
||||
let mut had_ingest_failures = false;
|
||||
let mut hit_cap_boundary = false;
|
||||
|
||||
for page_num in 0..MAX_PAGES_PER_SYNC {
|
||||
for page_num in 0..effective_max_pages {
|
||||
if state.budget_exhausted() {
|
||||
tracing::info!(
|
||||
page = page_num,
|
||||
@@ -249,9 +277,27 @@ impl ComposioProvider for NotionProvider {
|
||||
}
|
||||
|
||||
// ── Step 4a: dedupe + decide which pages to ingest ──────
|
||||
let (pending, hit_cursor_boundary) =
|
||||
let (mut pending, mut hit_cursor_boundary) =
|
||||
select_pending(&results, &state, &mut newest_edited_time);
|
||||
|
||||
// ctx.sync_depth_days: drop items edited before the depth floor. `pending` is
|
||||
// in descending timestamp order, so truncate at the first item below the floor
|
||||
// and signal cursor-boundary so pagination stops.
|
||||
if let Some(ref floor) = oldest_allowed_time {
|
||||
if let Some(cut) = pending.iter().position(|p| {
|
||||
p.edited_time
|
||||
.as_deref()
|
||||
.map(|t| t < floor.as_str())
|
||||
.unwrap_or(false)
|
||||
}) {
|
||||
pending.truncate(cut);
|
||||
hit_cursor_boundary = true;
|
||||
}
|
||||
}
|
||||
|
||||
// ctx.max_items: clamp the dedup'd batch to the remaining budget before ingest.
|
||||
cap.clamp_batch(&mut pending);
|
||||
|
||||
// ── Step 4b: ingest queued pages (bounded concurrency) ──
|
||||
let ingestor = MemoryTreeIngestor {
|
||||
config: ctx.config.as_ref(),
|
||||
@@ -262,10 +308,17 @@ impl ComposioProvider for NotionProvider {
|
||||
state.mark_synced(key);
|
||||
}
|
||||
total_persisted += outcome.persisted;
|
||||
cap.record(outcome.persisted);
|
||||
if outcome.had_failures {
|
||||
had_ingest_failures = true;
|
||||
}
|
||||
|
||||
// ctx.max_items precise cap: once the per-source cap is hit, stop paginating.
|
||||
if cap.is_reached() {
|
||||
hit_cap_boundary = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if hit_cursor_boundary {
|
||||
tracing::debug!(
|
||||
page = page_num,
|
||||
@@ -274,6 +327,17 @@ impl ComposioProvider for NotionProvider {
|
||||
break;
|
||||
}
|
||||
|
||||
// ctx.max_items hard stop.
|
||||
if cap.is_reached() {
|
||||
tracing::debug!(
|
||||
page = page_num,
|
||||
total_persisted,
|
||||
"[composio:notion] [memory_sync] max_items reached, stopping pagination"
|
||||
);
|
||||
hit_cap_boundary = true;
|
||||
break;
|
||||
}
|
||||
|
||||
// Check for next page cursor from Notion API.
|
||||
notion_cursor = sync::extract_notion_cursor(&resp.data);
|
||||
if notion_cursor.is_none() {
|
||||
@@ -287,15 +351,18 @@ impl ComposioProvider for NotionProvider {
|
||||
// Hold the cursor when any item failed to ingest this pass. See the
|
||||
// `had_ingest_failures` declaration above for why this matters under
|
||||
// the delete-first memory-tree pipeline (#2885).
|
||||
if !had_ingest_failures {
|
||||
// Hold the cursor on a cap-truncated pass so the next sync re-scans the unseen tail.
|
||||
if !had_ingest_failures && !hit_cap_boundary {
|
||||
if let Some(new_cursor) = newest_edited_time {
|
||||
state.advance_cursor(&new_cursor);
|
||||
}
|
||||
} else {
|
||||
tracing::warn!(
|
||||
connection_id = %connection_id,
|
||||
"[composio:notion] holding cursor — ingest failures this pass; next sync will \
|
||||
re-fetch the failed range"
|
||||
had_ingest_failures,
|
||||
hit_cap_boundary,
|
||||
"[composio:notion] holding cursor — ingest failures or cap-truncated pass; next \
|
||||
sync will re-fetch the failed range"
|
||||
);
|
||||
}
|
||||
state.save(&memory).await?;
|
||||
|
||||
@@ -468,6 +468,13 @@ impl ComposioProvider for SlackProvider {
|
||||
let mut total_messages_ingested: usize = 0;
|
||||
let mut channels_processed: usize = 0;
|
||||
let mut channels_errored: usize = 0;
|
||||
let mut hit_cap_boundary = false;
|
||||
|
||||
// ctx.max_items: ItemCap is threaded through process_channel so the
|
||||
// per-page batch is clamped before ingest and the channel loop stops
|
||||
// precisely at the cap — the old coarse post-channel check allowed a
|
||||
// single page/channel to blow past the cap.
|
||||
let mut cap = super::super::helpers::ItemCap::new(ctx.max_items);
|
||||
|
||||
// 2. Per-channel: fetch → post-process → enrich → ingest.
|
||||
for channel in &channels {
|
||||
@@ -488,6 +495,7 @@ impl ComposioProvider for SlackProvider {
|
||||
now,
|
||||
&users,
|
||||
&connection_id,
|
||||
&mut cap,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -506,6 +514,27 @@ impl ComposioProvider for SlackProvider {
|
||||
}
|
||||
}
|
||||
|
||||
// ctx.max_items hard stop across all channels (precise — cap was
|
||||
// already applied inside process_channel so this break fires
|
||||
// exactly when the budget is exhausted, not one channel later).
|
||||
if cap.is_reached() {
|
||||
// Hold the cursor on a cap-truncated pass so the next sync re-scans the unseen tail.
|
||||
hit_cap_boundary = true;
|
||||
tracing::debug!(
|
||||
connection_id = %connection_id,
|
||||
total_messages_ingested,
|
||||
"[composio:slack] [memory_sync] max_items reached, stopping channel iteration"
|
||||
);
|
||||
// Save state before breaking without advancing the cursor.
|
||||
if let Err(err) = state.save(&memory).await {
|
||||
tracing::warn!(
|
||||
error = %err,
|
||||
"[composio:slack] state save failed after cap-stop (non-fatal)"
|
||||
);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
state.advance_cursor(sync::encode_cursors(&cursors));
|
||||
if let Err(err) = state.save(&memory).await {
|
||||
tracing::warn!(
|
||||
@@ -515,6 +544,15 @@ impl ComposioProvider for SlackProvider {
|
||||
}
|
||||
}
|
||||
|
||||
if hit_cap_boundary {
|
||||
// Hold the cursor on a cap-truncated pass so the next sync re-scans the unseen tail.
|
||||
tracing::warn!(
|
||||
connection_id = %connection_id,
|
||||
"[composio:slack] cap-truncated pass; cursor held so next sync re-scans the \
|
||||
unseen tail"
|
||||
);
|
||||
}
|
||||
|
||||
let finished_at_ms = sync::now_ms();
|
||||
let summary = format!(
|
||||
"slack sync: channels_processed={channels_processed} \
|
||||
@@ -608,7 +646,12 @@ async fn list_all_channels(
|
||||
}
|
||||
|
||||
/// Pull one channel's history since its cursor, post-process + enrich each
|
||||
/// page, then ingest all messages. Returns the number of chunks written.
|
||||
/// page, then ingest all messages. Returns the number of messages written.
|
||||
///
|
||||
/// `cap` is the shared [`super::super::helpers::ItemCap`] for the sync pass.
|
||||
/// Each page's message batch is clamped to the remaining budget before ingest
|
||||
/// so the per-sync `max_items` limit is respected precisely regardless of how
|
||||
/// many messages a single channel/page returns.
|
||||
async fn process_channel(
|
||||
ctx: &ProviderContext,
|
||||
state: &mut SyncState,
|
||||
@@ -617,13 +660,26 @@ async fn process_channel(
|
||||
now: chrono::DateTime<chrono::Utc>,
|
||||
users: &SlackUsers,
|
||||
connection_id: &str,
|
||||
cap: &mut super::super::helpers::ItemCap,
|
||||
) -> Result<usize, String> {
|
||||
// Cursor value is a raw Slack `ts` (`"<seconds>.<micro>"`) preserved
|
||||
// with full precision, so multi-message-per-second channels don't
|
||||
// replay the whole second on the next incremental fetch. When no
|
||||
// cursor exists yet, fall back to `<backfill_window_secs>.000000`.
|
||||
// ctx.sync_depth_days wins over the env-var OPENHUMAN_SLACK_BACKFILL_DAYS
|
||||
// default when set — it comes from the user-configured source entry.
|
||||
let oldest_ts = cursors.get(&channel.id).cloned().unwrap_or_else(|| {
|
||||
let secs = (now - chrono::Duration::days(backfill_days())).timestamp();
|
||||
let depth_days = ctx
|
||||
.sync_depth_days
|
||||
.map(|d| d as i64)
|
||||
.unwrap_or_else(backfill_days);
|
||||
let secs = (now - chrono::Duration::days(depth_days)).timestamp();
|
||||
tracing::debug!(
|
||||
channel = %channel.id,
|
||||
depth_days,
|
||||
oldest_ts_secs = secs,
|
||||
"[composio:slack] [memory_sync] computing oldest_ts for backfill"
|
||||
);
|
||||
format!("{secs}.000000")
|
||||
});
|
||||
|
||||
@@ -676,6 +732,23 @@ async fn process_channel(
|
||||
break;
|
||||
}
|
||||
all_messages.extend(msgs);
|
||||
|
||||
// Stop fetching further pages for this channel if we have already
|
||||
// accumulated enough to fill the remaining budget (checked against
|
||||
// remaining() which accounts for items recorded by previous channels).
|
||||
if let Some(remaining) = cap.remaining() {
|
||||
if all_messages.len() >= remaining {
|
||||
tracing::debug!(
|
||||
channel = %channel.id,
|
||||
page = page_num,
|
||||
accumulated = all_messages.len(),
|
||||
remaining,
|
||||
"[composio:slack] [memory_sync] budget nearly full, stopping history pagination"
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
cursor = sync::extract_next_cursor(&resp.data);
|
||||
if cursor.is_none() {
|
||||
break;
|
||||
@@ -690,6 +763,19 @@ async fn process_channel(
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
// ctx.max_items precise cap: clamp the full accumulated batch to the
|
||||
// remaining budget before ingest so we never persist more than the cap
|
||||
// allows, even if a single channel/page returned more than what remains.
|
||||
cap.clamp_batch(&mut all_messages);
|
||||
|
||||
if all_messages.is_empty() {
|
||||
tracing::debug!(
|
||||
channel = %channel.id,
|
||||
"[composio:slack] [memory_sync] cap already reached, skipping channel ingest"
|
||||
);
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
let msg_count = all_messages.len();
|
||||
tracing::info!(
|
||||
channel = %channel.id,
|
||||
@@ -713,13 +799,16 @@ async fn process_channel(
|
||||
{
|
||||
cursors.insert(channel.id.clone(), latest);
|
||||
}
|
||||
cap.record(msg_count);
|
||||
tracing::info!(
|
||||
channel = %channel.id,
|
||||
messages = msg_count,
|
||||
chunks,
|
||||
"[composio:slack] channel ingest done"
|
||||
);
|
||||
Ok(chunks)
|
||||
// Return message count (consistent with the sync path which
|
||||
// counts messages, not chunks, for the items_ingested metric).
|
||||
Ok(msg_count)
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
|
||||
@@ -111,6 +111,8 @@ pub async fn sync_trigger_rpc(
|
||||
toolkit: conn.toolkit.clone(),
|
||||
connection_id: Some(conn.id.clone()),
|
||||
usage: Default::default(),
|
||||
max_items: None,
|
||||
sync_depth_days: None,
|
||||
};
|
||||
match provider.sync(&ctx, SyncReason::Manual).await {
|
||||
Ok(o) => outcomes.push(o),
|
||||
|
||||
@@ -295,6 +295,18 @@ pub struct ProviderContext {
|
||||
/// (`run_connection_sync`) reads it back. Non-sync callers (agent tools,
|
||||
/// task-source fetches) leave it at zero — harmless.
|
||||
pub usage: ComposioUsageHandle,
|
||||
/// Maximum items to fetch in a single sync pass.
|
||||
///
|
||||
/// Set from the corresponding `MemorySourceEntry.max_items` field at
|
||||
/// sync-dispatch time. `None` means no cap beyond the provider's own
|
||||
/// internal upper bounds.
|
||||
pub max_items: Option<u32>,
|
||||
/// Maximum sync depth window in days.
|
||||
///
|
||||
/// Set from `MemorySourceEntry.sync_depth_days`. When `Some(n)`, the
|
||||
/// provider only fetches items from the last `n` days. `None` means
|
||||
/// no additional depth restriction beyond the provider's cursor.
|
||||
pub sync_depth_days: Option<u32>,
|
||||
}
|
||||
|
||||
impl ProviderContext {
|
||||
@@ -325,6 +337,8 @@ impl ProviderContext {
|
||||
toolkit: toolkit.into(),
|
||||
connection_id,
|
||||
usage: ComposioUsageHandle::default(),
|
||||
max_items: None,
|
||||
sync_depth_days: None,
|
||||
}),
|
||||
Err(e) => {
|
||||
tracing::debug!(
|
||||
@@ -487,6 +501,8 @@ mod tests {
|
||||
toolkit: "gmail".to_string(),
|
||||
connection_id: None,
|
||||
usage: ComposioUsageHandle::default(),
|
||||
max_items: None,
|
||||
sync_depth_days: None,
|
||||
};
|
||||
let cloned = ctx.clone();
|
||||
|
||||
@@ -542,6 +558,8 @@ mod tests {
|
||||
toolkit: "gmail".to_string(),
|
||||
connection_id: None,
|
||||
usage: ComposioUsageHandle::default(),
|
||||
max_items: None,
|
||||
sync_depth_days: None,
|
||||
};
|
||||
let res = ctx.execute("GMAIL_FETCH_EMAILS", None).await;
|
||||
// The actual HTTP call will fail in the unit-test sandbox, but
|
||||
@@ -575,6 +593,8 @@ mod tests {
|
||||
toolkit: "gmail".to_string(),
|
||||
connection_id: None,
|
||||
usage: ComposioUsageHandle::default(),
|
||||
max_items: None,
|
||||
sync_depth_days: None,
|
||||
};
|
||||
let res = ctx.execute("GMAIL_FETCH_EMAILS", None).await;
|
||||
let err = res.expect_err("no backend session must error");
|
||||
|
||||
@@ -153,6 +153,8 @@ pub async fn preview_filter(
|
||||
toolkit: provider.as_str().to_string(),
|
||||
connection_id: connection_id.filter(|s| !s.trim().is_empty()),
|
||||
usage: Default::default(),
|
||||
max_items: None,
|
||||
sync_depth_days: None,
|
||||
};
|
||||
let max = max.unwrap_or(config.task_sources.max_tasks_per_fetch);
|
||||
let fetch_filter = filter::to_fetch_filter(&filter_spec, max);
|
||||
@@ -179,6 +181,8 @@ pub async fn list_databases(
|
||||
toolkit: provider.as_str().to_string(),
|
||||
connection_id: connection_id.filter(|s| !s.trim().is_empty()),
|
||||
usage: Default::default(),
|
||||
max_items: None,
|
||||
sync_depth_days: None,
|
||||
};
|
||||
let databases = provider_impl
|
||||
.list_databases(&ctx)
|
||||
|
||||
@@ -102,6 +102,8 @@ async fn run_inner(
|
||||
toolkit: source.provider.as_str().to_string(),
|
||||
connection_id: source.connection_id.clone(),
|
||||
usage: Default::default(),
|
||||
max_items: None,
|
||||
sync_depth_days: None,
|
||||
};
|
||||
|
||||
let fetch_filter = filter::to_fetch_filter(&source.filter, source.max_tasks_per_fetch);
|
||||
|
||||
@@ -2856,6 +2856,7 @@ async fn worker_a_controller_schemas_are_fully_exposed() {
|
||||
"memory_sources",
|
||||
vec![
|
||||
"openhuman.memory_sources_add",
|
||||
"openhuman.memory_sources_apply_all_in",
|
||||
"openhuman.memory_sources_estimate_sync_cost",
|
||||
"openhuman.memory_sources_get",
|
||||
"openhuman.memory_sources_list",
|
||||
|
||||
@@ -25,6 +25,9 @@ use openhuman_core::openhuman::memory_sync::composio::bus::{
|
||||
use openhuman_core::openhuman::memory_sync::composio::providers::clickup::ClickUpProvider;
|
||||
use openhuman_core::openhuman::memory_sync::composio::providers::github::GitHubProvider;
|
||||
use openhuman_core::openhuman::memory_sync::composio::providers::gmail::ingest as gmail_ingest;
|
||||
use openhuman_core::openhuman::memory_sync::composio::providers::gmail::GmailProvider;
|
||||
use openhuman_core::openhuman::memory_sync::composio::providers::linear::LinearProvider;
|
||||
use openhuman_core::openhuman::memory_sync::composio::providers::notion::NotionProvider;
|
||||
use openhuman_core::openhuman::memory_sync::composio::providers::slack::ingest as slack_ingest;
|
||||
use openhuman_core::openhuman::memory_sync::composio::providers::slack::{
|
||||
SlackMessage, SlackProvider,
|
||||
@@ -300,6 +303,8 @@ async fn configured_loopback_context(
|
||||
toolkit: toolkit.to_string(),
|
||||
connection_id: Some(connection_id.to_string()),
|
||||
usage: Default::default(),
|
||||
max_items: None,
|
||||
sync_depth_days: None,
|
||||
};
|
||||
(config, ctx, server)
|
||||
}
|
||||
@@ -552,6 +557,8 @@ async fn github_clickup_and_composio_bus_cover_provider_branches() {
|
||||
toolkit: "clickup".to_string(),
|
||||
connection_id: Some("conn-clickup-round17".to_string()),
|
||||
usage: Default::default(),
|
||||
max_items: None,
|
||||
sync_depth_days: None,
|
||||
};
|
||||
let clickup = ClickUpProvider::new();
|
||||
let click_profile = clickup
|
||||
@@ -624,6 +631,572 @@ async fn github_clickup_and_composio_bus_cover_provider_branches() {
|
||||
server.abort();
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Slack cap enforcement
|
||||
//
|
||||
// Proves that max_items=N caps Slack ingestion to exactly N messages even
|
||||
// when a single channel/page returns more than N messages. The mock returns
|
||||
// 5 messages in one channel; with max_items=2 only 2 must be persisted.
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Build a loopback router for the Slack cap test. Returns 5 messages for
|
||||
/// the single channel when `SLACK_FETCH_CONVERSATION_HISTORY` is called;
|
||||
/// all other Slack bootstrap calls (auth, users, channel listing) return
|
||||
/// minimal but valid responses.
|
||||
fn slack_cap_router(requests: Arc<Mutex<Vec<Value>>>) -> Router {
|
||||
Router::new().route(
|
||||
"/agent-integrations/composio/execute",
|
||||
any(move |Json(body): Json<Value>| {
|
||||
let requests = Arc::clone(&requests);
|
||||
async move {
|
||||
requests.lock().unwrap().push(body.clone());
|
||||
let tool = body.get("tool").and_then(Value::as_str).unwrap_or("");
|
||||
let resp = match tool {
|
||||
"SLACK_TEST_AUTH" => execute_envelope(json!({
|
||||
"user_id": "UCAP",
|
||||
"user": "capuser",
|
||||
"team": "Cap Workspace",
|
||||
"team_id": "TCAP",
|
||||
"url": "https://cap.slack.com"
|
||||
})),
|
||||
"SLACK_RETRIEVE_DETAILED_USER_INFORMATION" => execute_envelope(json!({
|
||||
"user": {
|
||||
"real_name": "Cap User",
|
||||
"profile": { "email": "cap@example.test" }
|
||||
}
|
||||
})),
|
||||
"SLACK_FETCH_TEAM_INFO" => execute_envelope(json!({
|
||||
"team": { "email_domain": "example.test" }
|
||||
})),
|
||||
// User directory — one member, no next page.
|
||||
"SLACK_LIST_ALL_USERS" => execute_envelope(json!({
|
||||
"members": [
|
||||
{ "id": "UCAP", "name": "capuser", "profile": { "real_name": "Cap User" } }
|
||||
],
|
||||
"response_metadata": { "next_cursor": "" }
|
||||
})),
|
||||
// One channel, no next page.
|
||||
"SLACK_LIST_CONVERSATIONS" => execute_envelope(json!({
|
||||
"channels": [
|
||||
{ "id": "CCAP", "name": "cap-channel", "is_private": false }
|
||||
],
|
||||
"response_metadata": { "next_cursor": "" }
|
||||
})),
|
||||
// Return 5 distinct messages for the single channel;
|
||||
// the cap is 2 so only 2 must be persisted.
|
||||
"SLACK_FETCH_CONVERSATION_HISTORY" => execute_envelope(json!({
|
||||
"messages": [
|
||||
{ "ts": "1800000001.000001", "user": "UCAP", "text": "cap message 1",
|
||||
"permalink": "https://cap.slack.com/archives/CCAP/p1800000001000001" },
|
||||
{ "ts": "1800000002.000002", "user": "UCAP", "text": "cap message 2",
|
||||
"permalink": "https://cap.slack.com/archives/CCAP/p1800000002000002" },
|
||||
{ "ts": "1800000003.000003", "user": "UCAP", "text": "cap message 3",
|
||||
"permalink": "https://cap.slack.com/archives/CCAP/p1800000003000003" },
|
||||
{ "ts": "1800000004.000004", "user": "UCAP", "text": "cap message 4",
|
||||
"permalink": "https://cap.slack.com/archives/CCAP/p1800000004000004" },
|
||||
{ "ts": "1800000005.000005", "user": "UCAP", "text": "cap message 5",
|
||||
"permalink": "https://cap.slack.com/archives/CCAP/p1800000005000005" }
|
||||
],
|
||||
"response_metadata": { "next_cursor": "" }
|
||||
})),
|
||||
_ => execute_envelope(json!({})),
|
||||
};
|
||||
Json(resp)
|
||||
}
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn slack_sync_max_items_caps_ingest_to_exact_count() {
|
||||
let _guard = env_lock();
|
||||
let tmp = TempDir::new().expect("tempdir");
|
||||
let _workspace = EnvGuard::set_path("OPENHUMAN_WORKSPACE", tmp.path());
|
||||
let _home = EnvGuard::set_path("HOME", tmp.path());
|
||||
let _backend = EnvGuard::unset("BACKEND_URL");
|
||||
// Disable inter-call pacing so the test runs quickly.
|
||||
let _pacing = EnvGuard::set("OPENHUMAN_SLACK_INTER_CALL_PACING_MS", "0");
|
||||
|
||||
let requests: Arc<Mutex<Vec<Value>>> = Arc::new(Mutex::new(Vec::new()));
|
||||
let (base, server) = loopback_router(slack_cap_router(Arc::clone(&requests))).await;
|
||||
|
||||
let mut config = config_in(&tmp);
|
||||
config.api_url = Some(base);
|
||||
persist_config(&config).await;
|
||||
store_session(&config);
|
||||
memory_global::init(config.workspace_dir.clone()).expect("init global memory");
|
||||
|
||||
let ctx = ProviderContext {
|
||||
config: Arc::new(config),
|
||||
toolkit: "slack".to_string(),
|
||||
connection_id: Some("conn-slack-cap".to_string()),
|
||||
usage: Default::default(),
|
||||
// The mock returns 5 messages; cap is 2 — only 2 must be ingested.
|
||||
max_items: Some(2),
|
||||
sync_depth_days: None,
|
||||
};
|
||||
|
||||
let outcome = SlackProvider::new()
|
||||
.sync(&ctx, SyncReason::ConnectionCreated)
|
||||
.await
|
||||
.expect("slack cap sync");
|
||||
|
||||
assert_eq!(
|
||||
outcome.items_ingested, 2,
|
||||
"max_items=2 must cap Slack ingest to exactly 2 even though the channel/page held 5"
|
||||
);
|
||||
|
||||
server.abort();
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Gmail cap enforcement
|
||||
//
|
||||
// Proves that max_items=N caps Gmail ingestion to exactly N messages even
|
||||
// when a single GMAIL_FETCH_EMAILS page returns more than N. The mock
|
||||
// returns 5 messages in one page; with max_items=2 only 2 must be
|
||||
// persisted.
|
||||
//
|
||||
// Gmail messages arrive in Composio's "upstream" shape before post-process
|
||||
// reshapes them into the slim envelope:
|
||||
// - messageId → id (used by MESSAGE_ID_PATHS dedup)
|
||||
// - sender → from (used by ingest bucketing)
|
||||
// - messageText → markdown body (from extract_markdown_body fallback)
|
||||
// - messageTimestamp → date
|
||||
// All 5 messages are unique (different messageId) and valid (non-empty
|
||||
// messageText), so without the cap every one would be ingested.
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Build M distinct, valid Gmail messages in the upstream (pre-post-process)
|
||||
/// Composio shape. Uses `messageId` / `sender` / `messageText` /
|
||||
/// `messageTimestamp` so `reshape_message` maps them correctly into the slim
|
||||
/// envelope that `ingest_page_into_memory_tree` expects.
|
||||
fn gmail_cap_messages(m: usize) -> Vec<Value> {
|
||||
(1..=m)
|
||||
.map(|i| {
|
||||
json!({
|
||||
"messageId": format!("gmail-cap-msg-{i}"),
|
||||
"threadId": format!("thread-cap-{i}"),
|
||||
"sender": format!("sender{i}@cap.example.test"),
|
||||
"to": "recipient@cap.example.test",
|
||||
"subject": format!("Cap test message {i}"),
|
||||
"messageTimestamp": format!("2026-06-0{}T10:00:00Z", (i % 9) + 1),
|
||||
"messageText": format!("Body of cap test message {i}. Sufficient content.")
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Loopback router that answers the two Gmail sync tool calls.
|
||||
/// GMAIL_GET_PROFILE returns a minimal valid profile; GMAIL_FETCH_EMAILS
|
||||
/// returns one page containing all `messages` and no next-page token.
|
||||
fn gmail_cap_router(messages: Vec<Value>, requests: Arc<Mutex<Vec<Value>>>) -> Router {
|
||||
Router::new().route(
|
||||
"/agent-integrations/composio/execute",
|
||||
any(move |Json(body): Json<Value>| {
|
||||
let messages = messages.clone();
|
||||
let requests = Arc::clone(&requests);
|
||||
async move {
|
||||
requests.lock().unwrap().push(body.clone());
|
||||
let tool = body.get("tool").and_then(Value::as_str).unwrap_or("");
|
||||
let resp = match tool {
|
||||
"GMAIL_GET_PROFILE" => execute_envelope(json!({
|
||||
"emailAddress": "cap@example.test",
|
||||
"messagesTotal": messages.len()
|
||||
})),
|
||||
"GMAIL_FETCH_EMAILS" => execute_envelope(json!({
|
||||
"messages": messages,
|
||||
"nextPageToken": ""
|
||||
})),
|
||||
_ => execute_envelope(json!({})),
|
||||
};
|
||||
Json(resp)
|
||||
}
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn gmail_sync_max_items_caps_ingest_to_exact_count() {
|
||||
let _guard = env_lock();
|
||||
let tmp = TempDir::new().expect("tempdir");
|
||||
let _workspace = EnvGuard::set_path("OPENHUMAN_WORKSPACE", tmp.path());
|
||||
let _home = EnvGuard::set_path("HOME", tmp.path());
|
||||
let _backend = EnvGuard::unset("BACKEND_URL");
|
||||
|
||||
// One page returns 5 valid messages; the cap is 2.
|
||||
let requests: Arc<Mutex<Vec<Value>>> = Arc::new(Mutex::new(Vec::new()));
|
||||
let (base, server) = loopback_router(gmail_cap_router(
|
||||
gmail_cap_messages(5),
|
||||
Arc::clone(&requests),
|
||||
))
|
||||
.await;
|
||||
|
||||
let mut config = config_in(&tmp);
|
||||
config.api_url = Some(base);
|
||||
persist_config(&config).await;
|
||||
store_session(&config);
|
||||
memory_global::init(config.workspace_dir.clone()).expect("init global memory");
|
||||
|
||||
let ctx = ProviderContext {
|
||||
config: Arc::new(config),
|
||||
toolkit: "gmail".to_string(),
|
||||
connection_id: Some("conn-gmail-cap".to_string()),
|
||||
usage: Default::default(),
|
||||
// Mock returns 5 messages; cap is 2 — only 2 must be ingested.
|
||||
max_items: Some(2),
|
||||
sync_depth_days: None,
|
||||
};
|
||||
|
||||
let outcome = GmailProvider::new()
|
||||
.sync(&ctx, SyncReason::ConnectionCreated)
|
||||
.await
|
||||
.expect("gmail cap sync");
|
||||
|
||||
assert_eq!(
|
||||
outcome.items_ingested, 2,
|
||||
"max_items=2 must cap Gmail ingest to exactly 2 even though the page held 5"
|
||||
);
|
||||
|
||||
server.abort();
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Notion cap enforcement
|
||||
//
|
||||
// Proves that max_items=N caps Notion ingestion to exactly N pages even
|
||||
// when a single NOTION_FETCH_DATA page returns more than N. The mock
|
||||
// returns 5 pages; with max_items=2 only 2 must be persisted.
|
||||
//
|
||||
// sync_depth_days is None and all items carry a recent last_edited_time
|
||||
// so no depth-filter skipping happens — the cap is the only thing that
|
||||
// limits ingestion.
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Build M distinct, valid Notion page objects. Each has a unique id and a
|
||||
/// recent `last_edited_time` so the depth filter (when enabled) would keep
|
||||
/// them all.
|
||||
fn notion_cap_pages(m: usize) -> Vec<Value> {
|
||||
(1..=m)
|
||||
.map(|i| {
|
||||
json!({
|
||||
"id": format!("notion-cap-page-{i:04}"),
|
||||
"object": "page",
|
||||
"last_edited_time": format!("2026-06-0{}T10:00:00.000Z", (i % 9) + 1),
|
||||
"properties": {
|
||||
"Name": {
|
||||
"type": "title",
|
||||
"title": [{ "plain_text": format!("Cap page {i}") }]
|
||||
}
|
||||
},
|
||||
"url": format!("https://www.notion.so/cap-page-{i}")
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Loopback router for the Notion cap test.
|
||||
/// NOTION_GET_ABOUT_ME returns a minimal identity; NOTION_FETCH_DATA returns
|
||||
/// one page with all `pages` as results and no next_cursor.
|
||||
fn notion_cap_router(pages: Vec<Value>, requests: Arc<Mutex<Vec<Value>>>) -> Router {
|
||||
Router::new().route(
|
||||
"/agent-integrations/composio/execute",
|
||||
any(move |Json(body): Json<Value>| {
|
||||
let pages = pages.clone();
|
||||
let requests = Arc::clone(&requests);
|
||||
async move {
|
||||
requests.lock().unwrap().push(body.clone());
|
||||
let tool = body.get("tool").and_then(Value::as_str).unwrap_or("");
|
||||
let resp = match tool {
|
||||
"NOTION_GET_ABOUT_ME" => execute_envelope(json!({
|
||||
"id": "notion-cap-user",
|
||||
"name": "Cap User",
|
||||
"type": "bot"
|
||||
})),
|
||||
"NOTION_FETCH_DATA" => execute_envelope(json!({
|
||||
"results": pages,
|
||||
"next_cursor": null,
|
||||
"has_more": false
|
||||
})),
|
||||
_ => execute_envelope(json!({})),
|
||||
};
|
||||
Json(resp)
|
||||
}
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn notion_sync_max_items_caps_ingest_to_exact_count() {
|
||||
let _guard = env_lock();
|
||||
let tmp = TempDir::new().expect("tempdir");
|
||||
let _workspace = EnvGuard::set_path("OPENHUMAN_WORKSPACE", tmp.path());
|
||||
let _home = EnvGuard::set_path("HOME", tmp.path());
|
||||
let _backend = EnvGuard::unset("BACKEND_URL");
|
||||
|
||||
// One page returns 5 valid Notion pages; the cap is 2.
|
||||
let requests: Arc<Mutex<Vec<Value>>> = Arc::new(Mutex::new(Vec::new()));
|
||||
let (base, server) = loopback_router(notion_cap_router(
|
||||
notion_cap_pages(5),
|
||||
Arc::clone(&requests),
|
||||
))
|
||||
.await;
|
||||
|
||||
let mut config = config_in(&tmp);
|
||||
config.api_url = Some(base);
|
||||
persist_config(&config).await;
|
||||
store_session(&config);
|
||||
memory_global::init(config.workspace_dir.clone()).expect("init global memory");
|
||||
|
||||
let ctx = ProviderContext {
|
||||
config: Arc::new(config),
|
||||
toolkit: "notion".to_string(),
|
||||
connection_id: Some("conn-notion-cap".to_string()),
|
||||
usage: Default::default(),
|
||||
// Mock returns 5 pages; cap is 2 — only 2 must be ingested.
|
||||
max_items: Some(2),
|
||||
sync_depth_days: None,
|
||||
};
|
||||
|
||||
let outcome = NotionProvider::new()
|
||||
.sync(&ctx, SyncReason::ConnectionCreated)
|
||||
.await
|
||||
.expect("notion cap sync");
|
||||
|
||||
assert_eq!(
|
||||
outcome.items_ingested, 2,
|
||||
"max_items=2 must cap Notion ingest to exactly 2 even though the page held 5"
|
||||
);
|
||||
|
||||
server.abort();
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Linear cap enforcement
|
||||
//
|
||||
// Proves that max_items=N caps Linear ingestion to exactly N issues even
|
||||
// when a single LINEAR_LIST_LINEAR_ISSUES page returns more than N.
|
||||
//
|
||||
// Tool sequence:
|
||||
// 1. LINEAR_LIST_LINEAR_USERS { isMe: true } → viewer id
|
||||
// 2. LINEAR_LIST_LINEAR_ISSUES (assigneeId=...) → nodes of issues
|
||||
//
|
||||
// Issues are in Linear's `{ nodes: [...], pageInfo: {...} }` shape.
|
||||
// We return hasNextPage=false so pagination stops after one page.
|
||||
// Each issue has a unique id and a recent updatedAt.
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Build M distinct, valid Linear issue objects.
|
||||
fn linear_cap_issues(m: usize) -> Vec<Value> {
|
||||
(1..=m)
|
||||
.map(|i| {
|
||||
json!({
|
||||
"id": format!("linear-cap-issue-{i:04}"),
|
||||
"identifier": format!("ENG-{i}"),
|
||||
"title": format!("Cap issue {i}"),
|
||||
"updatedAt": format!("2026-06-0{}T10:00:00.000Z", (i % 9) + 1),
|
||||
"url": format!("https://linear.app/cap/issue/ENG-{i}"),
|
||||
"description": format!("Description for cap issue {i}.")
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Loopback router for the Linear cap test.
|
||||
/// LINEAR_LIST_LINEAR_USERS returns a single viewer node so `resolve_viewer_id`
|
||||
/// succeeds. LINEAR_LIST_LINEAR_ISSUES returns one page with all issues and no
|
||||
/// next-page cursor (hasNextPage=false).
|
||||
fn linear_cap_router(issues: Vec<Value>, requests: Arc<Mutex<Vec<Value>>>) -> Router {
|
||||
Router::new().route(
|
||||
"/agent-integrations/composio/execute",
|
||||
any(move |Json(body): Json<Value>| {
|
||||
let issues = issues.clone();
|
||||
let requests = Arc::clone(&requests);
|
||||
async move {
|
||||
requests.lock().unwrap().push(body.clone());
|
||||
let tool = body.get("tool").and_then(Value::as_str).unwrap_or("");
|
||||
let resp = match tool {
|
||||
"LINEAR_LIST_LINEAR_USERS" => execute_envelope(json!({
|
||||
"nodes": [
|
||||
{ "id": "linear-cap-viewer", "name": "Cap Viewer", "email": "cap@linear.test" }
|
||||
]
|
||||
})),
|
||||
"LINEAR_LIST_LINEAR_ISSUES" => execute_envelope(json!({
|
||||
"nodes": issues,
|
||||
"pageInfo": {
|
||||
"hasNextPage": false,
|
||||
"endCursor": null
|
||||
}
|
||||
})),
|
||||
_ => execute_envelope(json!({})),
|
||||
};
|
||||
Json(resp)
|
||||
}
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn linear_sync_max_items_caps_ingest_to_exact_count() {
|
||||
let _guard = env_lock();
|
||||
let tmp = TempDir::new().expect("tempdir");
|
||||
let _workspace = EnvGuard::set_path("OPENHUMAN_WORKSPACE", tmp.path());
|
||||
let _home = EnvGuard::set_path("HOME", tmp.path());
|
||||
let _backend = EnvGuard::unset("BACKEND_URL");
|
||||
|
||||
// One page returns 5 valid Linear issues; the cap is 2.
|
||||
let requests: Arc<Mutex<Vec<Value>>> = Arc::new(Mutex::new(Vec::new()));
|
||||
let (base, server) = loopback_router(linear_cap_router(
|
||||
linear_cap_issues(5),
|
||||
Arc::clone(&requests),
|
||||
))
|
||||
.await;
|
||||
|
||||
let mut config = config_in(&tmp);
|
||||
config.api_url = Some(base);
|
||||
persist_config(&config).await;
|
||||
store_session(&config);
|
||||
memory_global::init(config.workspace_dir.clone()).expect("init global memory");
|
||||
|
||||
let ctx = ProviderContext {
|
||||
config: Arc::new(config),
|
||||
toolkit: "linear".to_string(),
|
||||
connection_id: Some("conn-linear-cap".to_string()),
|
||||
usage: Default::default(),
|
||||
// Mock returns 5 issues; cap is 2 — only 2 must be ingested.
|
||||
max_items: Some(2),
|
||||
sync_depth_days: None,
|
||||
};
|
||||
|
||||
let outcome = LinearProvider::new()
|
||||
.sync(&ctx, SyncReason::ConnectionCreated)
|
||||
.await
|
||||
.expect("linear cap sync");
|
||||
|
||||
assert_eq!(
|
||||
outcome.items_ingested, 2,
|
||||
"max_items=2 must cap Linear ingest to exactly 2 even though the page held 5"
|
||||
);
|
||||
|
||||
server.abort();
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// ClickUp cap enforcement
|
||||
//
|
||||
// Proves that max_items=N caps ClickUp ingestion to exactly N tasks even
|
||||
// when a single CLICKUP_GET_FILTERED_TEAM_TASKS page returns more than N.
|
||||
//
|
||||
// Tool sequence:
|
||||
// 1. CLICKUP_GET_AUTHORIZED_USER → user numeric id
|
||||
// 2. CLICKUP_GET_AUTHORIZED_TEAMS_WORKSPACES → one workspace
|
||||
// 3. CLICKUP_GET_FILTERED_TEAM_TASKS → tasks page
|
||||
//
|
||||
// The tasks page returns 5 items; with max_items=2 only 2 must be
|
||||
// persisted. Because INITIAL_PAGE_SIZE=100 for ConnectionCreated and we
|
||||
// return only 5 tasks (< 100), the short-page guard stops the loop so no
|
||||
// second page is requested, matching the cap path cleanly.
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Build M distinct, valid ClickUp task objects.
|
||||
fn clickup_cap_tasks(m: usize) -> Vec<Value> {
|
||||
// date_updated must be large enough to sort lexicographically correctly
|
||||
// but recent enough not to trip any depth filter. We use ms-since-epoch
|
||||
// strings in the vicinity of mid-2026 (≈1780000000000 ms).
|
||||
(1..=m)
|
||||
.map(|i| {
|
||||
json!({
|
||||
"id": format!("clickup-cap-task-{i:04}"),
|
||||
"name": format!("Cap task {i}"),
|
||||
"text_content": format!("Content for cap task {i}."),
|
||||
"status": { "status": "open" },
|
||||
"date_updated": format!("{}", 1_780_000_000_000_u64 + i as u64 * 1000),
|
||||
"url": format!("https://app.clickup.com/t/clickup-cap-task-{i:04}")
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Loopback router for the ClickUp cap test.
|
||||
fn clickup_cap_router(tasks: Vec<Value>, requests: Arc<Mutex<Vec<Value>>>) -> Router {
|
||||
Router::new().route(
|
||||
"/agent-integrations/composio/execute",
|
||||
any(move |Json(body): Json<Value>| {
|
||||
let tasks = tasks.clone();
|
||||
let requests = Arc::clone(&requests);
|
||||
async move {
|
||||
requests.lock().unwrap().push(body.clone());
|
||||
let tool = body.get("tool").and_then(Value::as_str).unwrap_or("");
|
||||
let resp = match tool {
|
||||
"CLICKUP_GET_AUTHORIZED_USER" => execute_envelope(json!({
|
||||
"user": {
|
||||
"id": 42,
|
||||
"username": "cap-user",
|
||||
"email": "cap@clickup.test",
|
||||
"profilePicture": null
|
||||
}
|
||||
})),
|
||||
"CLICKUP_GET_AUTHORIZED_TEAMS_WORKSPACES" => execute_envelope(json!({
|
||||
"teams": [
|
||||
{ "id": "ws-cap-01", "name": "Cap Workspace" }
|
||||
]
|
||||
})),
|
||||
"CLICKUP_GET_FILTERED_TEAM_TASKS" => execute_envelope(json!({
|
||||
"tasks": tasks
|
||||
})),
|
||||
_ => execute_envelope(json!({})),
|
||||
};
|
||||
Json(resp)
|
||||
}
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn clickup_sync_max_items_caps_ingest_to_exact_count() {
|
||||
let _guard = env_lock();
|
||||
let tmp = TempDir::new().expect("tempdir");
|
||||
let _workspace = EnvGuard::set_path("OPENHUMAN_WORKSPACE", tmp.path());
|
||||
let _home = EnvGuard::set_path("HOME", tmp.path());
|
||||
let _backend = EnvGuard::unset("BACKEND_URL");
|
||||
|
||||
// One workspace, one page returning 5 valid tasks; the cap is 2.
|
||||
let requests: Arc<Mutex<Vec<Value>>> = Arc::new(Mutex::new(Vec::new()));
|
||||
let (base, server) = loopback_router(clickup_cap_router(
|
||||
clickup_cap_tasks(5),
|
||||
Arc::clone(&requests),
|
||||
))
|
||||
.await;
|
||||
|
||||
let mut config = config_in(&tmp);
|
||||
config.api_url = Some(base);
|
||||
persist_config(&config).await;
|
||||
store_session(&config);
|
||||
memory_global::init(config.workspace_dir.clone()).expect("init global memory");
|
||||
|
||||
let ctx = ProviderContext {
|
||||
config: Arc::new(config),
|
||||
toolkit: "clickup".to_string(),
|
||||
connection_id: Some("conn-clickup-cap".to_string()),
|
||||
usage: Default::default(),
|
||||
// Mock returns 5 tasks; cap is 2 — only 2 must be ingested.
|
||||
max_items: Some(2),
|
||||
sync_depth_days: None,
|
||||
};
|
||||
|
||||
let outcome = ClickUpProvider::new()
|
||||
.sync(&ctx, SyncReason::ConnectionCreated)
|
||||
.await
|
||||
.expect("clickup cap sync");
|
||||
|
||||
assert_eq!(
|
||||
outcome.items_ingested, 2,
|
||||
"max_items=2 must cap ClickUp ingest to exactly 2 even though the page held 5"
|
||||
);
|
||||
|
||||
server.abort();
|
||||
}
|
||||
|
||||
fn walk_files(root: &Path) -> Vec<std::path::PathBuf> {
|
||||
let mut out = Vec::new();
|
||||
if !root.exists() {
|
||||
@@ -646,3 +1219,185 @@ fn walk_files(root: &Path) -> Vec<std::path::PathBuf> {
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Sync-cap enforcement: max_items and sync_depth_days
|
||||
//
|
||||
// These verify the user-facing promise of the per-source sync settings:
|
||||
// - max_items=N ingests AT MOST N items, even when a single API page
|
||||
// returns more (precise mid-page cap, not just a page cap).
|
||||
// - sync_depth_days injects an `updated:>{floor}` date filter into the
|
||||
// GitHub search query so only recent items are requested.
|
||||
// The cap logic is shared verbatim across the gmail/notion/linear/clickup
|
||||
// providers, so GitHub stands in for all of them here.
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Build `n` distinct, valid GitHub search items (each with an id + updated_at).
|
||||
fn github_issue_items(n: usize) -> Vec<Value> {
|
||||
(1..=n)
|
||||
.map(|i| {
|
||||
json!({
|
||||
"id": 3000 + i,
|
||||
"title": format!("Cap issue {i}"),
|
||||
"body": "cap enforcement body",
|
||||
"state": "open",
|
||||
"updated_at": format!("2026-05-{:02}T10:00:00Z", 10 + i),
|
||||
"html_url": format!("https://github.com/tinyhumansai/openhuman/issues/{i}")
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Loopback router that answers the two GitHub sync tools. The search tool
|
||||
/// always returns `items` (one page), and every request body is captured.
|
||||
fn github_cap_router(items: Vec<Value>, requests: Arc<Mutex<Vec<Value>>>) -> Router {
|
||||
Router::new().route(
|
||||
"/agent-integrations/composio/execute",
|
||||
any(move |Json(body): Json<Value>| {
|
||||
let items = items.clone();
|
||||
let requests = Arc::clone(&requests);
|
||||
async move {
|
||||
requests.lock().unwrap().push(body.clone());
|
||||
let tool = body.get("tool").and_then(Value::as_str).unwrap_or("");
|
||||
let resp = match tool {
|
||||
"GITHUB_GET_THE_AUTHENTICATED_USER" => execute_envelope(json!({
|
||||
"login": "octo-cap",
|
||||
"html_url": "https://github.com/octo-cap"
|
||||
})),
|
||||
"GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS" => execute_envelope(json!({
|
||||
"items": items,
|
||||
"total_count": items.len()
|
||||
})),
|
||||
_ => execute_envelope(json!({})),
|
||||
};
|
||||
Json(resp)
|
||||
}
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
async fn github_cap_context(
|
||||
tmp: &TempDir,
|
||||
items: Vec<Value>,
|
||||
requests: Arc<Mutex<Vec<Value>>>,
|
||||
) -> (Config, tokio::task::JoinHandle<()>) {
|
||||
let (base, server) = loopback_router(github_cap_router(items, requests)).await;
|
||||
let mut config = config_in(tmp);
|
||||
config.api_url = Some(base);
|
||||
persist_config(&config).await;
|
||||
store_session(&config);
|
||||
memory_global::init(config.workspace_dir.clone()).expect("init global memory");
|
||||
(config, server)
|
||||
}
|
||||
|
||||
fn github_ctx(
|
||||
config: &Config,
|
||||
max_items: Option<u32>,
|
||||
sync_depth_days: Option<u32>,
|
||||
) -> ProviderContext {
|
||||
ProviderContext {
|
||||
config: Arc::new(config.clone()),
|
||||
toolkit: "github".to_string(),
|
||||
connection_id: Some("conn-cap".to_string()),
|
||||
usage: Default::default(),
|
||||
max_items,
|
||||
sync_depth_days,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn github_sync_max_items_caps_ingest_to_exact_count() {
|
||||
let _guard = env_lock();
|
||||
let tmp = TempDir::new().expect("tempdir");
|
||||
let _workspace = EnvGuard::set_path("OPENHUMAN_WORKSPACE", tmp.path());
|
||||
let _home = EnvGuard::set_path("HOME", tmp.path());
|
||||
let _backend = EnvGuard::unset("BACKEND_URL");
|
||||
|
||||
// One page returns 5 valid items; the cap is 2.
|
||||
let requests = Arc::new(Mutex::new(Vec::new()));
|
||||
let (config, server) =
|
||||
github_cap_context(&tmp, github_issue_items(5), Arc::clone(&requests)).await;
|
||||
|
||||
let outcome = GitHubProvider::new()
|
||||
.sync(
|
||||
&github_ctx(&config, Some(2), None),
|
||||
SyncReason::ConnectionCreated,
|
||||
)
|
||||
.await
|
||||
.expect("github sync");
|
||||
|
||||
assert_eq!(
|
||||
outcome.items_ingested, 2,
|
||||
"max_items=2 must cap ingest to exactly 2 even though the page held 5"
|
||||
);
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn github_sync_without_max_items_ingests_full_page() {
|
||||
let _guard = env_lock();
|
||||
let tmp = TempDir::new().expect("tempdir");
|
||||
let _workspace = EnvGuard::set_path("OPENHUMAN_WORKSPACE", tmp.path());
|
||||
let _home = EnvGuard::set_path("HOME", tmp.path());
|
||||
let _backend = EnvGuard::unset("BACKEND_URL");
|
||||
|
||||
// Control: no cap → every valid item on the page is ingested.
|
||||
let requests = Arc::new(Mutex::new(Vec::new()));
|
||||
let (config, server) =
|
||||
github_cap_context(&tmp, github_issue_items(5), Arc::clone(&requests)).await;
|
||||
|
||||
let outcome = GitHubProvider::new()
|
||||
.sync(
|
||||
&github_ctx(&config, None, None),
|
||||
SyncReason::ConnectionCreated,
|
||||
)
|
||||
.await
|
||||
.expect("github sync");
|
||||
|
||||
assert_eq!(
|
||||
outcome.items_ingested, 5,
|
||||
"with no max_items cap, all 5 page items must be ingested"
|
||||
);
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn github_sync_depth_days_injects_updated_floor_into_query() {
|
||||
let _guard = env_lock();
|
||||
let tmp = TempDir::new().expect("tempdir");
|
||||
let _workspace = EnvGuard::set_path("OPENHUMAN_WORKSPACE", tmp.path());
|
||||
let _home = EnvGuard::set_path("HOME", tmp.path());
|
||||
let _backend = EnvGuard::unset("BACKEND_URL");
|
||||
|
||||
let requests = Arc::new(Mutex::new(Vec::new()));
|
||||
let (config, server) =
|
||||
github_cap_context(&tmp, github_issue_items(1), Arc::clone(&requests)).await;
|
||||
|
||||
GitHubProvider::new()
|
||||
.sync(
|
||||
&github_ctx(&config, None, Some(7)),
|
||||
SyncReason::ConnectionCreated,
|
||||
)
|
||||
.await
|
||||
.expect("github sync");
|
||||
|
||||
// The search request must carry an `updated:>{date}` floor derived from the
|
||||
// 7-day window, proving sync_depth_days actually narrows what is fetched.
|
||||
let reqs = requests.lock().unwrap();
|
||||
let search = reqs
|
||||
.iter()
|
||||
.find(|b| {
|
||||
b.get("tool").and_then(Value::as_str) == Some("GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS")
|
||||
})
|
||||
.expect("a search request was issued");
|
||||
let q = search
|
||||
.get("arguments")
|
||||
.and_then(|a| a.get("q"))
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("");
|
||||
assert!(
|
||||
q.contains("updated:>"),
|
||||
"sync_depth_days=7 must inject an `updated:>` date floor, got query: {q}"
|
||||
);
|
||||
server.abort();
|
||||
}
|
||||
|
||||
@@ -165,6 +165,8 @@ async fn configured_context(
|
||||
toolkit: toolkit.to_string(),
|
||||
connection_id: Some(connection_id.to_string()),
|
||||
usage: Default::default(),
|
||||
max_items: None,
|
||||
sync_depth_days: None,
|
||||
};
|
||||
(config, ctx, server)
|
||||
}
|
||||
|
||||
@@ -260,6 +260,8 @@ async fn configured_loopback_context(
|
||||
toolkit: "slack".to_string(),
|
||||
connection_id: Some("conn-slack-round19".to_string()),
|
||||
usage: Default::default(),
|
||||
max_items: None,
|
||||
sync_depth_days: None,
|
||||
};
|
||||
(config, ctx, server)
|
||||
}
|
||||
|
||||
@@ -508,6 +508,8 @@ async fn composio_providers_fetch_profiles_tasks_and_cover_error_branches() {
|
||||
toolkit: "github".to_string(),
|
||||
connection_id: Some("conn-github".to_string()),
|
||||
usage: Default::default(),
|
||||
max_items: None,
|
||||
sync_depth_days: None,
|
||||
};
|
||||
let github = GitHubProvider::new();
|
||||
let github_profile = github
|
||||
|
||||
@@ -239,6 +239,8 @@ async fn configured_loopback_context(
|
||||
toolkit: toolkit.to_string(),
|
||||
connection_id: Some(connection_id.to_string()),
|
||||
usage: Default::default(),
|
||||
max_items: None,
|
||||
sync_depth_days: None,
|
||||
};
|
||||
(config, ctx, server)
|
||||
}
|
||||
|
||||
@@ -3047,6 +3047,8 @@ async fn memory_sync_provider_trait_defaults_and_connection_hook_are_determinist
|
||||
toolkit: "raw_coverage".into(),
|
||||
connection_id: Some("conn-1".into()),
|
||||
usage: Default::default(),
|
||||
max_items: None,
|
||||
sync_depth_days: None,
|
||||
};
|
||||
let provider = RawCoverageProvider { fail_profile: true };
|
||||
assert_eq!(provider.sync_interval_secs(), Some(15 * 60));
|
||||
|
||||
@@ -599,6 +599,8 @@ async fn default_composio_provider_hooks_return_expected_noop_shapes() {
|
||||
toolkit: "round14".into(),
|
||||
connection_id: Some("conn-round14".into()),
|
||||
usage: Default::default(),
|
||||
max_items: None,
|
||||
sync_depth_days: None,
|
||||
};
|
||||
let provider = MinimalProvider;
|
||||
assert_eq!(provider.sync_interval_secs(), None);
|
||||
|
||||
Reference in New Issue
Block a user