diff --git a/docs/architecture/KEY_FILES.md b/docs/architecture/KEY_FILES.md index b40e6a45f..e4651b99c 100644 --- a/docs/architecture/KEY_FILES.md +++ b/docs/architecture/KEY_FILES.md @@ -302,8 +302,8 @@ per-release `**vX.Y.Z:**` narration — CI enforces this - `src/core/embedding-dim-check.ts` extension — facts.embedding dim drift surface. `readFactsEmbeddingDim(engine): Promise` covers both `vector(N)` and `halfvec(N)` shapes (migration v40 falls back to `vector` on pgvector < 0.7); regex ordering is halfvec-before-vector (substring "vec" appears in "halfvec"; naive `/vector/i` would shadow). `buildFactsAlterRecipe(dims, configured, type)` emits the paste-ready `DROP INDEX IF EXISTS idx_facts_embedding_hnsw; ALTER TABLE facts ALTER COLUMN embedding TYPE halfvec(N) USING embedding::halfvec(N); CREATE INDEX idx_facts_embedding_hnsw ON facts USING hnsw (embedding halfvec_cosine_ops) WHERE ...` flow (NOT bare REINDEX, which doesn't rewrite the index after a column-type change). `assertFactsEmbeddingDimMatchesConfig(engine)` is the preflight — throws `FactsEmbeddingDimMismatchError` (tagged `tag: 'FACTS_EMBEDDING_DIM_MISMATCH'` for parity with the worker-pool MUST_ABORT semantics) when configured dim ≠ column width; cached per-engine via `WeakMap`; PGLite engines silently skip. Doctor check `facts_embedding_width_consistency` (registered after `embedding_width_consistency`) reuses the same helpers with an identical ALTER recipe. Pinned by `test/embedding-dim-check-facts.test.ts`. - `src/core/postgres-engine.ts` extension — `insertFact` + `insertFacts` no longer hardcode `tx.unsafe(\`'${embedLit}'::vector\`)` for the embedding column. `resolveFactsEmbeddingCast()` (private) probes `pg_attribute` once per engine instance (cached in `_factsEmbeddingCastSuffix`) and returns `'::halfvec'` when migration v40 created the column as halfvec, else `'::vector'`; both insert paths use the cached suffix so the cast matches the actual column type (works on older pgvector that lacks implicit auto-cast). Test seam `__resetFactsEmbeddingCastCacheForTest()` clears the per-engine cache. - `src/core/cycle.ts` + `src/core/cycle/extract-atoms.ts` + `src/core/cycle/synthesize-concepts.ts` + `src/commands/extract.ts` + `src/commands/doctor.ts` + `src/core/op-checkpoint.ts` extensions — six daily-driver ops fixes. (1) Batch idempotency: `atomsExistingForHashes(engine, sourceId, hashes[])` (exported from `src/core/cycle/extract-atoms.ts`) replaces the per-hash loop (7K individual queries) with one batched SQL roundtrip returning already-extracted `content_hash16` values; fail-open (SQL error → empty set, extraction proceeds); powered by migration v104 `pages_atom_source_hash_idx` (partial expression index on `frontmatter->>'source_hash'` for atom rows where `deleted_at IS NULL`; Postgres `CREATE INDEX CONCURRENTLY` with invalid-remnant pre-drop, PGLite plain). (2) Shorter cycle lock TTL + active in-phase refresh: `LOCK_TTL_MINUTES = 5` (was 30); `buildYieldDuringPhase(lock, outer)` (exported, with `LockHandle`) calls `lock.refresh()` + any external hook on every fire, throttled to 30s via `maybeYield`, firing both in the main loop AND immediately after every `await chat(...)`; `synthesize_concepts` uses the same throttled hook. A crashed cycle releases its lock 6x faster while a healthy long-running cycle keeps it alive (residual: a single `await chat()` past 5 min can expire the lock mid-await — TODO-OPS-2). (3) Progress wiring: `progress?: ProgressReporter` opt on `ExtractAtomsOpts` and `SynthesizeConceptsOpts`; cycle.ts passes its phase-level reporter down (NOT a child reporter, which would collide on `cycle.extract_atoms.extract_atoms.work`); phases only call `tick()`/`heartbeat()`, cycle.ts owns `start()`/`finish()`. (4) `by-mention` resume: `mentionsFingerprint({source, type, since, gazetteerHash})` in `src/core/op-checkpoint.ts` — the gazetteer hash is load-bearing (adding entity pages mid-pause shifts the hash → new fingerprint → fresh scan against the new gazetteer, never silent skip); `gbrain extract links --by-mention` resumes via `op_checkpoints` with `flushAndCheckpoint` ordering (links flush to DB FIRST, page keys commit to checkpoint SECOND, persist THIRD, so a crash mid-batch leaves the page un-checkpointed and resume re-scans it); persist every 1000 items OR 30s; clean exit clears the checkpoint; `--dry-run` skips both load and write. (5) `sync_consolidation` doctor check (multi-source brains see a paste-ready `gbrain sync --all --parallel 4 --workers 4 --skip-failed`; single-source "not applicable"; SQL errors return `warn` via the check's own try/catch). (6) Test-isolation: `test/cycle-last-full-cycle-at.test.ts` + `test/schema-cli.test.ts` use per-test `GBRAIN_HOME=tempdir`. Pinned by `test/cycle/extract-atoms-batch.test.ts`, `test/cycle/cycle-lock-ttl.test.ts` (regression pin on `LOCK_TTL_MINUTES === 5`), `test/op-checkpoint-mentions-fingerprint.test.ts`, `test/cycle/extract-atoms-progress.test.ts`, `test/cycle/synthesize-concepts-progress.test.ts`, `test/cycle/yield-during-phase-refresh.test.ts`, `test/cycle/yield-during-phase-throttle.test.ts`, `test/extract-by-mention-resume.test.ts`, `test/doctor-sync-consolidation.test.ts`. Companion `sync --all` recipe block in `skills/cron-scheduler/SKILL.md`. -- `src/commands/sync.ts` — `gbrain sync` CLI + the `performSync` / `performFullSync` library entrypoints (consumed by the autopilot cycle and the Minion sync handler). `performSync` runs under a writer lock: per-source `gbrain-sync:` whenever `opts.sourceId` is set, wrapped in `withRefreshingLock` from `src/core/db-lock.ts` so long-running sources (250K+ chunks) don't lose the lock at the 30-min TTL mid-run; the bare no-source path uses the SAME refreshing lock (#1794 — it was previously a non-refreshing `tryAcquireDbLock`, stealable mid-run during an incident); `SyncOpts.lockId?: string` is the explicit override. The lock refresh AND its health probe route through the DIRECT session pool so Supavisor transaction-pool exhaustion (`EMAXCONNSESSION`) can't kill renewal; takeover is heartbeat-aware (it will NOT steal a holder whose `last_refreshed_at` is within `GBRAIN_LOCK_STEAL_GRACE_SECONDS`, defending an alive-but-starved holder); the import loop yields the event loop every `GBRAIN_SYNC_YIELD_EVERY` files (`setTimeout(0)`, not `setImmediate` — Bun starves the timers phase) so the refresh `setInterval` heartbeat fires mid-import. This lock-identity invariant prevents a `sync --all` per-source worker racing `sync --source foo` on the global lock from corrupting the same source. `performSync` throws a typed `SyncLockBusyError` when the writer lock is held; the Minion `sync` handler (`src/commands/jobs.ts`) catches it and marks the job SKIPPED (not failed) so a cron/autopilot tick defers to the holder without polluting crash metrics. `performSyncInner` is RESUMABLE (incremental path): it drains a PINNED target commit (`lastCommit..pin`), banking drained file paths via `appendCompleted` (append-only delta into the `op_checkpoint_paths` child table, migration v115 — one row per path, O(delta) not the old O(N²) full-array rewrite), keyed by `syncFingerprint({sourceId, lastCommit})` from `src/core/op-checkpoint.ts` (paths under `op:'sync'`; the pinned target under `op:'sync-target'`), and advances `last_commit`/`last_sync_at` ONLY at full import completion. Checkpoint writes route through the DIRECT session pool + bounded retry so they survive `EMAXCONNSESSION`; the flush cadence is first-file then every `GBRAIN_SYNC_CHECKPOINT_EVERY` (default 1000) files OR `GBRAIN_SYNC_CHECKPOINT_SECONDS` (default 10s), with a race-safe `pendingCheckpointPaths` delta (single-flight swap, re-merge on failure) under parallel workers; a SIGTERM banks the in-flight delta via a no-retry one-shot (`appendCompletedOnce`, ordered before lock release through `registerCleanup`); and sustained flush failure aborts the run with `reason:'checkpoint_unavailable'` after `GBRAIN_SYNC_MAX_CHECKPOINT_FAILURES` consecutive fails rather than importing work it can never bank (every partial/blocked exit logs the banked-file count). A sync killed mid-import banks its progress, leaves the anchor unmoved (the source stays correctly stale to the autopilot scheduler — `last_sync_at` is never bumped on a partial), and the next run `resumeFilter`s the same fixed diff to skip done files. The pin is the checkpoint's stored target when still reachable from HEAD, else live HEAD (a history rewrite / reset re-pins); completion advances to the pin, NOT live HEAD, so commits landing past the pin are a clean next-sync diff — this closes the cross-run staleness window. After import a pin-reachability gate (`git merge-base --is-ancestor pin HEAD`) replaces the old strict head-drift gate: forward commits on top of the pin (e.g. a background enrich process committing to the same repo every ~2 min) are SAFE and no longer block the run; only a real rewrite (pin not an ancestor of HEAD) blocks and discards the checkpoint. A file added in `lastCommit..pin` but gone from disk (deleted by a commit after the pin) is SKIPPED and marked completed, not recorded as a failure. Downstream extract/facts/embed are size-gated: inline only for `totalChanges <= 100`; large syncs defer to the resumable `extract --stale` watermark + `embed --stale`/backfill + the facts cycle phases, so a 44K-page facts/embed pass never re-blocks import convergence (sync convergence == import convergence). Worker engines wrap in try/finally so disconnect always fires; both PGLite-detection sites use `engine.kind === 'pglite'`. CLI accepts `--workers N` (alias `--concurrency N`) validated via `parseWorkers` (explicit bypasses the file-count floor; auto path defers to `autoConcurrency()`). The newest-first descending-lex order uses `sortNewestFirst(addsAndMods)` from `src/core/sort-newest-first.ts` (shared with `gbrain import`). `gbrain sync --all` runs a continuous worker pool: `parseWorkers`-validated `--parallel N` (default `min(sourceCount, --workers, DEFAULT_PARALLEL_SOURCES=4)`), long-lived async workers pulling from a shared FIFO queue (no head-of-line blocking), per-source `withSourcePrefix(src.id, ...)` so every `slog`/`serr` line carries `[]`; `--skip-failed`/`--retry-failed` are scoped per source (`acknowledgeFailures(sourceId)`; `--all` acks every source, single-source acks only its own) and run UNDER parallel — the #1939 failure ledger is per-`(source_id, path)` and serialized through `withLedgerLock`, so the old "not supported under parallel, re-run with --serial" refusal is lifted (#2139, which also removed the forcing-function that pushed recovery syncs to `--serial` and thus armed the inline cost gate); a connection-budget stderr warning fires when `parallel × workers × 2 > 16` (the `× 2 per-file pool` factor: each per-file worker opens its own `PostgresEngine` with `poolSize=2`). Exports `resolveParallelism`, `syncOneSource`, `buildSyncStatusReport`, `printSyncStatusReport`, `SyncStatusReport` back the `gbrain sources status` dashboard. `--json` envelope `{schema_version: 1, sources, parallel, ok_count, error_count}` on stdout; human banners route to stderr via `humanSink` so `jq` parses cleanly. Exit matrix: 0 all ok, 1 any error. (The non-TTY cost gate no longer exits 2 — it auto-defers; #2139.) The dashboard SQL is `content_chunks ch JOIN pages pg ON pg.id = ch.page_id WHERE pg.deleted_at IS NULL` with `archived = false` at the caller; embedding column resolved via `resolveEmbeddingColumn(undefined, cfg)` from `src/core/search/embedding-column.ts` so Voyage/multimodal/non-default-column brains count against the column they use; errors propagate (no swallow-catch). The sync delete loop is interleaved per-batch resolve+delete using `engine.resolveSlugsByPaths` + `engine.deletePages` from `src/core/engine.ts` (73K-delete commit: ~146K SQL round-trips → ~292, closing the cascade-staleness class where one big-delete commit jammed every other source's sync); per-batch try-catch decomposes batch DELETE failures to per-slug `deletePage` fallback, unrecoverable per-slug failures land in `failedFiles`; `pagesAffected` filters to confirmed-deleted slugs. The rename loop gets the same batched slug-resolve; a rename whose destination is un-syncable folds the source path into the delete set so the old page can't orphan (#1970). An entry-time bookmark-reachability guard distinguishes a gc'd anchor (`cat-file` fails → `performFullSync`) from a history-rewrite anchor that is merely no-longer-an-ancestor but still on disk: the latter is diffed tree-to-tree directly (`git diff lastCommit..pin` is an endpoint-tree compare, ancestry not required) so a force-push / `master`→`main` consolidation imports only the real delta instead of re-walking the whole tree forever (#1970); an oversized or failed diff degrades to `performFullSync`. `performFullSync` is itself authoritative for deletes — after an advancing full import it purges file-backed pages (`source_path != null` AND strategy-aware `isSyncable`) whose source file no longer exists, sparing `put_page`/manual pages (null `source_path`) and metafiles. The stale-file decision routes through the pure, exported `planReconcileDeletes(rows, currentFiles, isSyncablePath)`: it normalizes path separators on both sides of the membership test (a Windows `path.relative` backslash path vs a git-derived forward-slash `source_path` would otherwise mark every page stale and wipe the source) and computes a mass-delete signal — when the reconcile would delete more than `MASS_RECONCILE_RATIO` (50%) of the file-backed pages the strategy manages, on a source holding more than `MASS_RECONCILE_MIN_PAGES` (20) of them, the delete is REFUSED with a loud stderr warning (that shape is almost always a path-comparison bug or the wrong repo path, not a genuine bulk deletion); `GBRAIN_ALLOW_MASS_RECONCILE=1` restores the unguarded delete for genuinely intended bulk removals. Pinned by `test/sync-reconcile-mass-delete.test.ts`. `resolveSlugByPathOrSourcePath` at `sync.ts:267` delegates to `engine.resolveSlugsByPaths` when `sourceId` is set, keeping legacy `executeRaw` fallback for the no-sourceId path. `failedFiles` is hoisted to the top of `performSyncInner` so both delete-decompose and import loops feed the same bookmark gate. The cost gate is the shared `runInlineCostGate` (one implementation on BOTH the `--all` and single-source paths; runs at the command layer, never inside `performSync`), mode-aware via `willEmbedSynchronously` + posture-aware `shouldBlockSync` from `src/core/embedding.ts` (#2139). The DEFERRED path (v2 on, parallel) is INFORMATIONAL (embedding goes to per-source `embed-backfill` jobs with their own `$X/source/24h` cap, default $25 via `SPEND_CAP_CONFIG_KEY` from `embed-backfill-submit.ts`; prints the cap + backlog + queued-job count, NEVER exits 2). The INLINE path (v2 off, or `--serial`) gates on the DELTA estimate vs `sync.cost_gate_min_usd` (default $0.50): below floor proceeds; above floor in a TTY prompts `[y/N]`; above floor in a non-TTY/`--json` session AUTO-DEFERS embeds to capped backfill jobs and exits 0 (NEVER exit 2 — the wedged-cron fix); `spend.posture=tokenmax` makes it informational and proceeds inline. The estimate MIRRORS EXECUTION instead of pricing the whole tree: `estimateInlineNewTokens` routes through the shared `computeSyncDelta` (`src/core/sync-delta.ts`) — fetch-first against `origin/`, prices only the committed delta (a dirty-but-caught-up tree → $0), with a full-tree CEILING only on the fail-open rungs (chunker drift, first sync, git-unavailable) honestly labeled; `--full` adds the stale backlog (full sync sweeps it inline). Return shape carries `estimateKind: 'delta'|'ceiling'|'mixed'|'unchanged'` + `ceilingReasons`. Helpers `resolveCostGateFloorUsd(engine)` + `resolveBackfillCapUsd(engine)` resolve via `parseUsdLimit` (`off`/`unlimited` → `Infinity`; floor accepts `0` = block-on-any-spend). JSON envelopes carry `mode` + `gate` discriminators (`dry_run | deferred_notice | below_floor | auto_deferred_embeds | posture_tokenmax`) + a paste-ready `hint`; `Infinity` floors/caps render as the string `'unlimited'` (never raw, which JSON-serializes to `null`); `SyncStatusReportSource` gains `backfill_queued`/`backfill_active`/`backfill_last_completed_at`; cost previews read `getEmbeddingModelName()` (no hardcoded OpenAI). Format splits on the explicit `--json` flag only (human text otherwise). `SyncOpts.noSchemaPack` (CLI `--no-schema-pack`, threaded through `performSync` AND `syncOneSource`) skips `loadActivePack` so pages fall back to legacy prefix typing — an escape hatch when a suspect pack regex wedges a sync. A per-file BEGIN heartbeat `if (process.env.GBRAIN_SYNC_TRACE) serr('[sync] begin import: ')` fires BEFORE `importFile` (the `progress.tick` fires only AFTER) so a stuck file is a begin-line with no matching completion. Triage doc: `docs/architecture/serve-sync-concurrency.md` (PGLite single-writer serve↔sync contention + the `GBRAIN_SYNC_TRACE` + `--no-schema-pack` recipes). Pinned by `test/e2e/sync-status-pglite.test.ts` (IRON-RULE: PGLite seeds 2 sources × pages × chunks, soft-deletes 1 page, archives 1 source, validates the SQL excludes both AND uses the active embedding column), `test/sync-cost-gate.serial.test.ts`, `test/sync-cost-preview.test.ts`. Runaway-sync protection: `resolveSyncHardDeadline(args, {isTty, env, defaultNonTtySec?})` resolves a wall-clock hard deadline (precedence `--no-hard-deadline` > `--hard-deadline ` > `--timeout `(non-`--all`, which auto-arms the backstop) > `GBRAIN_SYNC_MAX_RUNTIME_SECONDS` env > non-TTY default 3600s > none; `HARD_DEADLINE_GRACE_SEC=30`). `src/cli.ts` installs the out-of-band watchdog (see `src/core/process-watchdog.ts`) for the sync command BEFORE `connectEngine` and disposes it in the dispatch `finally`, so even an event-loop-starved sync — or a connect-phase hang — is SIGTERM-then-SIGKILLed by the deadline instead of orphaning under cron. `runSync` registers a SIGINT handler that aborts an interrupt `AbortController` composed via `composeAbortSignals(...)` (an `AbortSignal.any` wrapper over the defined signals) with the per-source `--timeout` signal, so Ctrl-C returns a clean `partial` and releases the lock through the normal `finally` (process-cleanup.ts owns SIGTERM lock-release; the watchdog owns the hard kill). `withRefreshingLock` `unref()`s its refresh `setInterval`. The spin's own root cause is not yet pinned (leading lead is catastrophic-backtracking in a pack link-inference regex, bounded by the redos-guard); the watchdog heartbeat plus the existing `[gbrain phase]` breadcrumbs are the diagnosis surface. Pinned by `test/sync-hard-deadline.test.ts` (resolution precedence + `composeAbortSignals`). -- `src/commands/import.ts` — `gbrain import` CLI + `runImport` library entrypoint. Uses a path-set checkpoint via `src/core/import-checkpoint.ts` (the walk still applies `sortNewestFirst()` for embed-cost ordering, but checkpoint correctness no longer depends on sort order). A file enters `completed: Set` only when its `processFile` returns success (including content-hash short-circuit no-ops); failed files never enter the set so the next run retries them automatically with no manual `~/.gbrain/import-checkpoint.json` delete. This closes three classes: parallel-import-with-slow-worker dropping the slow file on crash-resume (the slow file isn't in `completed` until its own `processFile` resolves), failed-file-bumps-counter-past-itself (failures don't add to `completed`), and sort-flip-drops-newest-N-on-cross-version-resume (order is no longer part of the checkpoint). Old positional checkpoints are detected and discarded with a stderr line on first resume (re-walking is cheap because `content_hash` short-circuits unchanged files). Checkpoint persists every 100 successful adds, not every 100 processed files. The `managedBookmark` opt (set by `performFullSync` when `runImport` is the full-sync engine) suppresses `runImport`'s own `sync.last_commit` advance so the shared `applySyncFailureGate` (`src/core/sync-failure-ledger.ts`) owns the bookmark + failure-ledger gating on that path — one gate decides advance/block/auto-skip across both sync paths. Pinned by `test/import-checkpoint.test.ts` + `test/import-resume.test.ts` (incl. the SLUG_MISMATCH retry regression). +- `src/commands/sync.ts` — `gbrain sync` CLI + the `performSync` / `performFullSync` library entrypoints (consumed by the autopilot cycle and the Minion sync handler). `performSync` runs under a writer lock: per-source `gbrain-sync:` whenever `opts.sourceId` is set, wrapped in `withRefreshingLock` from `src/core/db-lock.ts` so long-running sources (250K+ chunks) don't lose the lock at the 30-min TTL mid-run; the bare no-source path uses the SAME refreshing lock (#1794 — it was previously a non-refreshing `tryAcquireDbLock`, stealable mid-run during an incident); `SyncOpts.lockId?: string` is the explicit override. The lock refresh AND its health probe route through the DIRECT session pool so Supavisor transaction-pool exhaustion (`EMAXCONNSESSION`) can't kill renewal; takeover is heartbeat-aware (it will NOT steal a holder whose `last_refreshed_at` is within `GBRAIN_LOCK_STEAL_GRACE_SECONDS`, defending an alive-but-starved holder); the import loop yields the event loop every `GBRAIN_SYNC_YIELD_EVERY` files (`setTimeout(0)`, not `setImmediate` — Bun starves the timers phase) so the refresh `setInterval` heartbeat fires mid-import. This lock-identity invariant prevents a `sync --all` per-source worker racing `sync --source foo` on the global lock from corrupting the same source. `performSync` throws a typed `SyncLockBusyError` when the writer lock is held; the Minion `sync` handler (`src/commands/jobs.ts`) catches it and marks the job SKIPPED (not failed) so a cron/autopilot tick defers to the holder without polluting crash metrics. `performSyncInner` is RESUMABLE (incremental path): it drains a PINNED target commit (`lastCommit..pin`), banking drained file paths via `appendCompleted` (append-only delta into the `op_checkpoint_paths` child table, migration v115 — one row per path, O(delta) not the old O(N²) full-array rewrite), keyed by `syncFingerprint({sourceId, lastCommit})` from `src/core/op-checkpoint.ts` (paths under `op:'sync'`; the pinned target under `op:'sync-target'`), and advances `last_commit`/`last_sync_at` ONLY at full import completion. Checkpoint writes route through the DIRECT session pool + bounded retry so they survive `EMAXCONNSESSION`; the flush cadence is first-file then every `GBRAIN_SYNC_CHECKPOINT_EVERY` (default 1000) files OR `GBRAIN_SYNC_CHECKPOINT_SECONDS` (default 10s), with a race-safe `pendingCheckpointPaths` delta (single-flight swap, re-merge on failure) under parallel workers; a SIGTERM banks the in-flight delta via a no-retry one-shot (`appendCompletedOnce`, ordered before lock release through `registerCleanup`); and sustained flush failure aborts the run with `reason:'checkpoint_unavailable'` after `GBRAIN_SYNC_MAX_CHECKPOINT_FAILURES` consecutive fails rather than importing work it can never bank (every partial/blocked exit logs the banked-file count). A sync killed mid-import banks its progress, leaves the anchor unmoved (the source stays correctly stale to the autopilot scheduler — `last_sync_at` is never bumped on a partial), and the next run `resumeFilter`s the same fixed diff to skip done files. The pin is the checkpoint's stored target when still reachable from HEAD, else live HEAD (a history rewrite / reset re-pins); completion advances to the pin, NOT live HEAD, so commits landing past the pin are a clean next-sync diff — this closes the cross-run staleness window. After import a pin-reachability gate (`git merge-base --is-ancestor pin HEAD`) replaces the old strict head-drift gate: forward commits on top of the pin (e.g. a background enrich process committing to the same repo every ~2 min) are SAFE and no longer block the run; only a real rewrite (pin not an ancestor of HEAD) blocks and discards the checkpoint. A file added in `lastCommit..pin` but gone from disk (deleted by a commit after the pin) is SKIPPED and marked completed, not recorded as a failure. Downstream extract/facts/embed are size-gated: inline only for `totalChanges <= 100`; large syncs defer to the resumable `extract --stale` watermark + `embed --stale`/backfill + the facts cycle phases, so a 44K-page facts/embed pass never re-blocks import convergence (sync convergence == import convergence). Worker engines wrap in try/finally so disconnect always fires; both PGLite-detection sites use `engine.kind === 'pglite'`. CLI accepts `--workers N` (alias `--concurrency N`) validated via `parseWorkers` (explicit bypasses the file-count floor; auto path defers to `autoConcurrency()`). The newest-first descending-lex order uses `sortNewestFirst(addsAndMods)` from `src/core/sort-newest-first.ts` (shared with `gbrain import`). `gbrain sync --all` runs a continuous worker pool: `parseWorkers`-validated `--parallel N` (default `min(sourceCount, --workers, DEFAULT_PARALLEL_SOURCES=4)`), long-lived async workers pulling from a shared FIFO queue (no head-of-line blocking), per-source `withSourcePrefix(src.id, ...)` so every `slog`/`serr` line carries `[]`; `--skip-failed`/`--retry-failed` are scoped per source (`acknowledgeFailures(sourceId)`; `--all` acks every source, single-source acks only its own) and run UNDER parallel — the #1939 failure ledger is per-`(source_id, path)` and serialized through `withLedgerLock`, so the old "not supported under parallel, re-run with --serial" refusal is lifted (#2139, which also removed the forcing-function that pushed recovery syncs to `--serial` and thus armed the inline cost gate); a connection-budget stderr warning fires when `parallel × workers × 2 > 16` (the `× 2 per-file pool` factor: each per-file worker opens its own `PostgresEngine` with `poolSize=2`). Exports `resolveParallelism`, `syncOneSource`, `buildSyncStatusReport`, `printSyncStatusReport`, `SyncStatusReport` back the `gbrain sources status` dashboard. `--json` envelope `{schema_version: 1, sources, parallel, ok_count, error_count}` on stdout; human banners route to stderr via `humanSink` so `jq` parses cleanly. Exit matrix: 0 all ok, 1 any error. (The non-TTY cost gate no longer exits 2 — it auto-defers; #2139.) The dashboard SQL is `content_chunks ch JOIN pages pg ON pg.id = ch.page_id WHERE pg.deleted_at IS NULL` with `archived = false` at the caller; embedding column resolved via `resolveEmbeddingColumn(undefined, cfg)` from `src/core/search/embedding-column.ts` so Voyage/multimodal/non-default-column brains count against the column they use; errors propagate (no swallow-catch). The sync delete loop is interleaved per-batch resolve+delete using `engine.resolveSlugsByPaths` + `engine.deletePages` from `src/core/engine.ts` (73K-delete commit: ~146K SQL round-trips → ~292, closing the cascade-staleness class where one big-delete commit jammed every other source's sync); per-batch try-catch decomposes batch DELETE failures to per-slug `deletePage` fallback, unrecoverable per-slug failures land in `failedFiles`; `pagesAffected` filters to confirmed-deleted slugs. The rename loop gets the same batched slug-resolve; a rename whose destination is un-syncable folds the source path into the delete set so the old page can't orphan (#1970). An entry-time bookmark-reachability guard distinguishes a gc'd anchor (`cat-file` fails → `performFullSync`) from a history-rewrite anchor that is merely no-longer-an-ancestor but still on disk: the latter is diffed tree-to-tree directly (`git diff lastCommit..pin` is an endpoint-tree compare, ancestry not required) so a force-push / `master`→`main` consolidation imports only the real delta instead of re-walking the whole tree forever (#1970); an oversized or failed diff degrades to `performFullSync`. `performFullSync` is itself authoritative for deletes — after an advancing full import it purges file-backed pages (`source_path != null` AND strategy-aware `isSyncable`) whose source file no longer exists, sparing `put_page`/manual pages (null `source_path`) and metafiles. The stale-file decision routes through the pure, exported `planReconcileDeletes(rows, currentFiles, isSyncablePath)`: it normalizes path separators on both sides of the membership test (a Windows `path.relative` backslash path vs a git-derived forward-slash `source_path` would otherwise mark every page stale and wipe the source) and computes a mass-delete signal — when the reconcile would delete more than `MASS_RECONCILE_RATIO` (50%) of the file-backed pages the strategy manages, on a source holding more than `MASS_RECONCILE_MIN_PAGES` (20) of them, the delete is REFUSED with a loud stderr warning (that shape is almost always a path-comparison bug or the wrong repo path, not a genuine bulk deletion); `GBRAIN_ALLOW_MASS_RECONCILE=1` restores the unguarded delete for genuinely intended bulk removals. Pinned by `test/sync-reconcile-mass-delete.test.ts`. `resolveSlugByPathOrSourcePath` at `sync.ts:267` delegates to `engine.resolveSlugsByPaths` when `sourceId` is set, keeping legacy `executeRaw` fallback for the no-sourceId path. `failedFiles` is hoisted to the top of `performSyncInner` so both delete-decompose and import loops feed the same bookmark gate. The cost gate is the shared `runInlineCostGate` (one implementation on BOTH the `--all` and single-source paths; runs at the command layer, never inside `performSync`), mode-aware via `willEmbedSynchronously` + posture-aware `shouldBlockSync` from `src/core/embedding.ts` (#2139). The DEFERRED path (v2 on, parallel) is INFORMATIONAL (embedding goes to per-source `embed-backfill` jobs with their own `$X/source/24h` cap, default $25 via `SPEND_CAP_CONFIG_KEY` from `embed-backfill-submit.ts`; prints the cap + backlog + queued-job count, NEVER exits 2). The INLINE path (v2 off, or `--serial`) gates on the DELTA estimate vs `sync.cost_gate_min_usd` (default $0.50): below floor proceeds; above floor in a TTY prompts `[y/N]`; above floor in a non-TTY/`--json` session AUTO-DEFERS embeds to capped backfill jobs and exits 0 (NEVER exit 2 — the wedged-cron fix); `spend.posture=tokenmax` makes it informational and proceeds inline. The estimate MIRRORS EXECUTION instead of pricing the whole tree: `estimateInlineNewTokens` routes through the shared `computeSyncDelta` (`src/core/sync-delta.ts`) — fetch-first against `origin/`, prices only the committed delta (a dirty-but-caught-up tree → $0), with a full-tree CEILING only on the fail-open rungs (chunker drift, first sync, git-unavailable) honestly labeled; `--full` adds the stale backlog (full sync sweeps it inline). Return shape carries `estimateKind: 'delta'|'ceiling'|'mixed'|'unchanged'` + `ceilingReasons`. Helpers `resolveCostGateFloorUsd(engine)` + `resolveBackfillCapUsd(engine)` resolve via `parseUsdLimit` (`off`/`unlimited` → `Infinity`; floor accepts `0` = block-on-any-spend). JSON envelopes carry `mode` + `gate` discriminators (`dry_run | deferred_notice | below_floor | auto_deferred_embeds | posture_tokenmax`) + a paste-ready `hint`; `Infinity` floors/caps render as the string `'unlimited'` (never raw, which JSON-serializes to `null`); `SyncStatusReportSource` gains `backfill_queued`/`backfill_active`/`backfill_last_completed_at`; cost previews read `getEmbeddingModelName()` (no hardcoded OpenAI). Format splits on the explicit `--json` flag only (human text otherwise). `SyncOpts.noSchemaPack` (CLI `--no-schema-pack`, threaded through `performSync` AND `syncOneSource`) skips `loadActivePack` so pages fall back to legacy prefix typing — an escape hatch when a suspect pack regex wedges a sync. A per-file BEGIN heartbeat `if (process.env.GBRAIN_SYNC_TRACE) serr('[sync] begin import: ')` fires BEFORE `importFile` (the `progress.tick` fires only AFTER) so a stuck file is a begin-line with no matching completion. Triage doc: `docs/architecture/serve-sync-concurrency.md` (PGLite single-writer serve↔sync contention + the `GBRAIN_SYNC_TRACE` + `--no-schema-pack` recipes). Pinned by `test/e2e/sync-status-pglite.test.ts` (IRON-RULE: PGLite seeds 2 sources × pages × chunks, soft-deletes 1 page, archives 1 source, validates the SQL excludes both AND uses the active embedding column), `test/sync-cost-gate.serial.test.ts`, `test/sync-cost-preview.test.ts`. Runaway-sync protection: `resolveSyncHardDeadline(args, {isTty, env, defaultNonTtySec?})` resolves a wall-clock hard deadline (precedence `--no-hard-deadline` > `--hard-deadline ` > `--timeout `(non-`--all`, which auto-arms the backstop) > `GBRAIN_SYNC_MAX_RUNTIME_SECONDS` env > non-TTY default 3600s > none; `HARD_DEADLINE_GRACE_SEC=30`). `src/cli.ts` installs the out-of-band watchdog (see `src/core/process-watchdog.ts`) for the sync command BEFORE `connectEngine` and disposes it in the dispatch `finally`, so even an event-loop-starved sync — or a connect-phase hang — is SIGTERM-then-SIGKILLed by the deadline instead of orphaning under cron. `runSync` registers a SIGINT handler that aborts an interrupt `AbortController` composed via `composeAbortSignals(...)` (an `AbortSignal.any` wrapper over the defined signals) with the per-source `--timeout` signal, so Ctrl-C returns a clean `partial` and releases the lock through the normal `finally` (process-cleanup.ts owns SIGTERM lock-release; the watchdog owns the hard kill). `withRefreshingLock` `unref()`s its refresh `setInterval`. The spin's own root cause is not yet pinned (leading lead is catastrophic-backtracking in a pack link-inference regex, bounded by the redos-guard); the watchdog heartbeat plus the existing `[gbrain phase]` breadcrumbs are the diagnosis surface. Pinned by `test/sync-hard-deadline.test.ts` (resolution precedence + `composeAbortSignals`). Monorepo subdir sources (#753/#774): `--src-subpath ` (or a repo path that IS a subdir — auto-discovery via `discoverGitRoot`, i.e. `git rev-parse --show-toplevel`) splits the repo path into `gitContextRoot` (all git ops: pull/diff/rev-parse/cat-file) and `syncScopeRoot` (walk/import/delete/rename scope); scoped syncs use git-root-relative slugs + `source_path` (full sync threads `slugRoot` into `runImport`) so full and incremental agree; NAV-1/NAV-2 realpath containment rejects `../`-traversal and symlinked scopes resolving outside the repo BEFORE any git op, and a per-file realpath guard (`isPathSafe`) refuses symlink-escape files in the incremental drain and rename reimport (fail-closed into `failedFiles`, so the bookmark can't advance past an escape); the full-sync reconcile is scope-restricted so a scoped sync never sweeps out-of-scope pages. `--exclude ` (repeatable) filters scope-relative paths in both full and incremental paths; exclusion never deletes previously-imported pages (conservative, matching the #1433 metafile posture); an all-excluded run warns loudly (NAV-4). Both flags are single-invocation only (rejected with `--all`; register the subdir as the source local_path instead). `.gitignore` management resolves to the git root via `manageGitignoreAtGitRoot`. Pinned by `test/sync-monorepo.test.ts`. +- `src/commands/import.ts` — `gbrain import` CLI + `runImport` library entrypoint. Uses a path-set checkpoint via `src/core/import-checkpoint.ts` (the walk still applies `sortNewestFirst()` for embed-cost ordering, but checkpoint correctness no longer depends on sort order). A file enters `completed: Set` only when its `processFile` returns success (including content-hash short-circuit no-ops); failed files never enter the set so the next run retries them automatically with no manual `~/.gbrain/import-checkpoint.json` delete. This closes three classes: parallel-import-with-slow-worker dropping the slow file on crash-resume (the slow file isn't in `completed` until its own `processFile` resolves), failed-file-bumps-counter-past-itself (failures don't add to `completed`), and sort-flip-drops-newest-N-on-cross-version-resume (order is no longer part of the checkpoint). Old positional checkpoints are detected and discarded with a stderr line on first resume (re-walking is cheap because `content_hash` short-circuits unchanged files). Checkpoint persists every 100 successful adds, not every 100 processed files. The `managedBookmark` opt (set by `performFullSync` when `runImport` is the full-sync engine) suppresses `runImport`'s own `sync.last_commit` advance so the shared `applySyncFailureGate` (`src/core/sync-failure-ledger.ts`) owns the bookmark + failure-ledger gating on that path — one gate decides advance/block/auto-skip across both sync paths. Pinned by `test/import-checkpoint.test.ts` + `test/import-resume.test.ts` (incl. the SLUG_MISMATCH retry regression). `runImport` opts also carry `exclude` (glob filter over dir-relative paths, threaded by `performFullSync` for `sync --exclude`; warns when every file is excluded — NAV-4) and `slugRoot` (slug/`source_path` base for monorepo subdir syncs, #753/#774; the resume checkpoint stays dir-relative per `resumeFilter`'s contract). - `src/core/import-checkpoint.ts` — `loadCheckpoint(brainDir)`, `saveCheckpoint(brainDir, completed)`, `resumeFilter(files, completed, brainDir)`, `clearCheckpoint()`, plus the `ImportCheckpoint` type. Path-set format `{schema_version, brainDir, completed: string[]}`. Atomic write via `.tmp` + `rename()` so a mid-write crash never leaves a partial JSON. `loadCheckpoint` returns `null` on: missing file, malformed JSON, brainDir mismatch (ran against a different brain), and the old positional format (logged to stderr before discard). `resumeFilter` returns `{toProcess, skippedCount}` — pure, no I/O, deterministic. `clearCheckpoint` is no-op-on-missing for clean-exit cleanup. Honors `GBRAIN_HOME` via `gbrainPath()` so `withEnv({GBRAIN_HOME: tmpdir})` test isolation works without monkey-patching fs. Best-effort persistence — `saveCheckpoint` logs warnings on write errors but never throws. - `src/core/sort-newest-first.ts` — single source of truth for the descending-lex sort that `gbrain import` and `gbrain sync` both apply. Mutates in place (Array.prototype.sort semantics), returns the same array reference for fluent chaining. Empty/single-element inputs short-circuit. Future ordering changes flip one line here instead of touching two CLI commands. Pinned by `test/sort-newest-first.test.ts` (descending order, mixed prefixes, empty, single-element, in-place-mutation contract). - `src/core/cycle.ts` — brain maintenance cycle primitive (9 phases). `runCycle(engine: BrainEngine | null, opts: CycleOpts): Promise` composes phases in semantic order: lint → backlinks → sync → synthesize → extract → patterns → recompute_emotional_weight → embed → orphans. `synthesize` runs after sync (cross-references see a fresh brain) and before extract (auto-link materializes its writes); `patterns` runs after extract so it reads a fresh graph (subagent put_page sets `ctx.remote=true` and skips auto-link/timeline by default, so extract is the canonical materialization); `recompute_emotional_weight` sees the union of `syncPagesAffected` + `synthesizeWrittenSlugs` incrementally, or all pages when neither anchor is set (full backfill via `gbrain dream --phase recompute_emotional_weight`). `CycleReport.schema_version: "1"` is stable; `totals` is additive (`pages_emotional_weight_recomputed`, `transcripts_processed`, `synth_pages_written`, `patterns_written`). Three callers: `gbrain dream` CLI, `gbrain autopilot` daemon inline path, the Minions `autopilot-cycle` handler. Coordination via `gbrain_cycle_locks` DB table + `~/.gbrain/cycle.lock` file lock with PID-liveness for PGLite. `yieldBetweenPhases` runs between phases; `yieldDuringPhase` is in-phase keepalive (synthesize/patterns renew the cycle-lock TTL during long waits). Engine nullable; lock-skip on read-only phase selections. `CycleOpts.signal?: AbortSignal` propagates the worker's abort signal with `checkAborted()` between every phase. `runPhaseSync` returns `pagesAffected` via `SyncPhaseResult` (threaded to `runPhaseExtract` as the 4th arg) and takes `willRunExtractPhase: boolean` setting `noExtract: phases.includes('extract')` so `gbrain dream --phase sync` doesn't silently lose extraction. `resolveSourceForDir(engine, brainDir)` threads `sourceId` to `performSync()` so sync reads the per-source `sources.last_commit` anchor (not the drift-prone global `config.sync.last_commit`). `CycleOpts.brainDir` is `string | null`; when null (checkout-less postgres/Supabase brain) the 6 filesystem phases (lint/backlinks/sync/synthesize/extract/patterns) skip with `details.reason: 'no_brain_dir'` and the DB-only phases run; `resolveSourceForDir` is null-tolerant. `cycleSourceId = opts.sourceId ?? resolveSourceForDir(engine, brainDir)` is the canonical per-source scope for `extract_facts`/`extract_atoms`/calibration so `gbrain dream --source repo-a` reconciles repo-a's facts even with no checkout (instead of scoping to `'default'` while stamping repo-a fresh). `deriveStatus` counts `edges_resolved`/`edges_ambiguous` as work so an edges-only cycle reports `ok` not `clean`; the `jobs.ts` `autopilot-cycle` + phase-wrapper handlers pass `null` (not `'.'`) when no repo is configured. The cycle is SPLIT for autopilot fan-out (#2194/#2227): `PHASE_SCOPE` partitions `ALL_PHASES` into `GLOBAL_PHASES` (brain-wide: embed/orphans/purge/resolve_symbol_edges/grade_takes/calibration_profile/synthesize_concepts/skillopt) and `NON_GLOBAL_PHASES` (source + mixed). Per-source `autopilot-cycle` jobs run only `NON_GLOBAL_PHASES` and stamp `last_source_cycle_at`; the single `autopilot-global-maintenance` job runs `GLOBAL_PHASES` (no `sourceId`) and stamps the brain-level `autopilot.last_global_at` config key (`LAST_GLOBAL_AT_KEY`). `last_full_cycle_at` is still written alongside `last_source_cycle_at` on a per-source success for doctor/legacy readers (no longer a gate for the brain-wide phases). Pinned by `test/dream-postgres.serial.test.ts` + `test/jobs-autopilot-cycle-braindir.serial.test.ts` + `test/autopilot-global-maintenance.test.ts`. diff --git a/src/commands/import.ts b/src/commands/import.ts index 1d2939318..9e792a384 100644 --- a/src/commands/import.ts +++ b/src/commands/import.ts @@ -11,6 +11,7 @@ import { isCodeFilePath, isMarkdownFilePath, isImageFilePath as isImageFilePathFromSync, + matchesAnyGlob, pruneDir, SYNC_SKIP_FILES, type SyncStrategy, @@ -47,7 +48,25 @@ export interface RunImportResult { export async function runImport( engine: BrainEngine, args: string[], - opts: { commit?: string; strategy?: SyncStrategy; sourceId?: string; managedBookmark?: boolean } = {}, + opts: { + commit?: string; + strategy?: SyncStrategy; + sourceId?: string; + managedBookmark?: boolean; + /** + * #753/#774: glob patterns to exclude from the import (same semantics as + * `isSyncable`'s `exclude` — matched against the dir-relative path). + * Threaded by performFullSync for `gbrain sync --exclude`. + */ + exclude?: string[]; + /** + * #753/#774 monorepo subdir-source support: when set, slugs and + * `source_path` are computed relative to this root (the git repo root) + * instead of `dir` (the sync scope), so `wiki/page1.md` lands as slug + * `wiki/page1` consistently across full and incremental sync. + */ + slugRoot?: string; + } = {}, ): Promise { const noEmbed = args.includes('--no-embed'); const fresh = args.includes('--fresh'); @@ -190,13 +209,30 @@ export async function runImport( const strategy: SyncStrategy = opts.strategy ?? 'markdown'; const _walkT0 = Date.now(); console.error(`[gbrain phase] import.collect_files start dir=${dir} strategy=${strategy}`); - const allFiles = collectSyncableFiles(dir, { strategy }); + let allFiles = collectSyncableFiles(dir, { strategy }); console.error( `[gbrain phase] import.collect_files done ${Date.now() - _walkT0}ms files=${allFiles.length}`, ); const fileTypeLabel = strategy === 'code' ? 'code' : strategy === 'auto' ? 'syncable' : 'markdown'; - console.log(`Found ${allFiles.length} ${fileTypeLabel} files`); + // #753/#774: apply --exclude glob patterns (threaded by performFullSync). + if (opts.exclude && opts.exclude.length > 0) { + const beforeExclude = allFiles.length; + allFiles = allFiles.filter(abs => !matchesAnyGlob(relative(dir, abs), opts.exclude)); + console.log( + `Found ${allFiles.length} ${fileTypeLabel} files ` + + `(${beforeExclude - allFiles.length} excluded by --exclude patterns)`, + ); + // NAV-4: everything excluded is almost always a mistyped pattern — warn. + if (beforeExclude > 0 && allFiles.length === 0) { + console.warn( + `[gbrain sync] No files matched after applying ${opts.exclude.length} --exclude pattern(s). ` + + `Check your --exclude flags. Patterns: ${JSON.stringify(opts.exclude)}`, + ); + } + } else { + console.log(`Found ${allFiles.length} ${fileTypeLabel} files`); + } // Sort newest-first so date-prefixed brain paths get embedded before older ones. // See src/core/sort-newest-first.ts for the policy. @@ -242,6 +278,11 @@ export async function runImport( async function processFile(eng: BrainEngine, filePath: string) { const relativePath = relative(dir, filePath); + // #753/#774: slug + source_path base. When performFullSync syncs a + // monorepo subdir, slugRoot is the git root so slugs stay git-root- + // relative (matching the incremental path's git-diff paths). The + // checkpoint (`completed`) stays dir-relative — resumeFilter's contract. + const importRelPath = opts.slugRoot ? relative(opts.slugRoot, filePath) : relativePath; // v0.31.2 (D5): per-file slow-path log. Fires only when a single // file takes >5s. The user's hang surfaces as one file taking // forever — without this, the agent can't see which file. @@ -252,8 +293,8 @@ export async function runImport( // up images when GBRAIN_EMBEDDING_MULTIMODAL=true so this branch is // unreachable when the gate is off; defense-in-depth check anyway. const result = isImageFilePath(relativePath) && process.env.GBRAIN_EMBEDDING_MULTIMODAL === 'true' - ? await importImageFile(eng, filePath, relativePath, { noEmbed, sourceId }) - : await importFile(eng, filePath, relativePath, { noEmbed, sourceId, activePack: importActivePack }); + ? await importImageFile(eng, filePath, importRelPath, { noEmbed, sourceId }) + : await importFile(eng, filePath, importRelPath, { noEmbed, sourceId, activePack: importActivePack }); const _fileMs = Date.now() - _fileT0; if (_fileMs > 5000) { console.error(`[gbrain phase] import.process_file slow ${_fileMs}ms ${relativePath}`); @@ -269,7 +310,9 @@ export async function runImport( if (result.error && result.error !== 'unchanged') { console.error(` Skipped ${relativePath}: ${result.error}`); // Bug 9 — non-"unchanged" skips carry a real error reason. - failures.push({ path: relativePath, error: result.error }); + // #774: ledger paths use the slug base so an incremental sync's + // success at the same (git-root-relative) path clears the row. + failures.push({ path: importRelPath, error: result.error }); } else { // 'unchanged' or no-error skip: content_hash matched a prior // successful import, so this file IS done for checkpoint purposes. @@ -287,7 +330,7 @@ export async function runImport( } errors++; skipped++; - failures.push({ path: relativePath, error: msg }); + failures.push({ path: importRelPath, error: msg }); } processed++; tickProgress(); diff --git a/src/commands/sync.ts b/src/commands/sync.ts index 9331a9a2f..0c2861934 100644 --- a/src/commands/sync.ts +++ b/src/commands/sync.ts @@ -1,4 +1,4 @@ -import { existsSync, readFileSync, writeFileSync, statSync } from 'fs'; +import { existsSync, readFileSync, writeFileSync, statSync, realpathSync } from 'fs'; import { execFileSync } from 'child_process'; import { join, relative } from 'path'; import type { BrainEngine } from '../core/engine.ts'; @@ -9,6 +9,7 @@ import { createInterface } from 'readline'; import { isSyncable, unsyncableReason, + matchesAnyGlob, resolveSlugForPath, unacknowledgedSyncFailures, acknowledgeFailures, @@ -742,6 +743,27 @@ export interface SyncOpts { sourceId?: string; /** Multi-repo: sync strategy override (markdown, code, auto). */ strategy?: 'markdown' | 'code' | 'auto'; + /** + * #753/#774 — sync only files under this subdirectory of the git repo. + * Git operations (pull, diff, rev-parse) still run against the repo root + * (discovered via `git rev-parse --show-toplevel`); file walking, imports, + * deletes and renames are scoped to the subpath. Slugs are git-root-relative + * (`wiki/page1.md` → slug `wiki/page1`) so full and incremental syncs of + * the same scope agree. Enables N logical sources in one git repo. + * + * SECURITY (NAV-1/NAV-2): the resolved subpath must realpath-resolve inside + * the git root — `../escape` and symlinked subdirs pointing outside the repo + * are rejected before any git op runs. + */ + srcSubpath?: string; + /** + * #753/#774 — glob patterns for files to exclude from sync (repeatable + * `--exclude` on the CLI). Matched against the scope-relative path in both + * the full-sync and incremental paths. Excluded files are never imported; + * exclusion does NOT delete previously-imported pages (conservative, + * matching the #1433 metafile posture). + */ + exclude?: string[]; /** * Number of parallel workers for the import phase. When > 1, each worker * gets its own small Postgres connection pool and files are dispatched via @@ -905,6 +927,37 @@ function git(repoPath: string, args: string[], configs: string[] = []): string { }).trim(); } +/** + * #753/#774: walk up from inputPath to the nearest git repo root via + * `git -C rev-parse --show-toplevel`. Handles worktrees and submodules + * natively (git itself resolves them). Throws a user-friendly error when no + * git repo is found. + */ +export function discoverGitRoot(inputPath: string): string { + try { + return git(inputPath, ['rev-parse', '--show-toplevel']); + } catch { + throw new Error( + `Not inside a git repository: ${inputPath}. GBrain sync requires a git-initialized repo (or a subdirectory of one).`, + ); + } +} + +/** + * #774 NAV-1 TOCTOU: true only if filePath realpath-resolves inside gitRoot. + * Guards symlink escape at the per-file level (a committed symlink whose + * target lives outside the repo), not just at scope entry. + */ +function isPathSafe(filePath: string, gitRoot: string): boolean { + try { + const real = realpathSync(filePath); + const rootReal = realpathSync(gitRoot); + return real === rootReal || real.startsWith(rootReal + '/'); + } catch { + return false; + } +} + function hasOriginRemote(repoPath: string): boolean { try { execFileSync('git', buildGitInvocation(repoPath, ['remote', 'get-url', 'origin']), { @@ -1567,17 +1620,46 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise 0 || detachedWorkingTreeManifest.modified.length > 0 || @@ -1814,7 +1897,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise + !scoped || p === syncScopeRelPath || p.startsWith(syncScopeRelPath + '/'); + // --exclude patterns match the SCOPE-relative path (what the user of a + // scoped source thinks in), same form runImport matches on full sync. + const scopeRel = (p: string): string => + scoped && p.startsWith(syncScopeRelPath + '/') ? p.slice(syncScopeRelPath.length + 1) : p; + const excluded = (p: string): boolean => + opts.exclude !== undefined && opts.exclude.length > 0 && matchesAnyGlob(scopeRel(p), opts.exclude); + + // Filter to syncable files (strategy-aware + scope-aware + exclude-aware) const syncOpts = opts.strategy ? { strategy: opts.strategy } : undefined; // #1970 (F-C): a rename whose DESTINATION is unsyncable drops out of BOTH // `renamed` (only `r.to` is kept below) AND `deleted` (git emits it as `R`, // not `D`), leaving the OLD page stale. Fold the source side into the delete // set. isSyncable(r.from) excludes metafiles automatically, so a rename of a // metafile is left untouched (matching the #1433 metafile-skip invariant). + // #774: a rename whose destination LEFT the scope is the same class — the + // old page's backing file is gone from this source's slice of the repo. const renamedToUnsyncable = manifest.renamed - .filter(r => isSyncable(r.from, syncOpts) && !isSyncable(r.to, syncOpts)) + .filter(r => inScope(r.from) && isSyncable(r.from, syncOpts) && + !(inScope(r.to) && isSyncable(r.to, syncOpts))) .map(r => r.from); const filtered: SyncManifest = { - added: manifest.added.filter(p => isSyncable(p, syncOpts)), - modified: manifest.modified.filter(p => isSyncable(p, syncOpts)), + added: manifest.added.filter(p => inScope(p) && !excluded(p) && isSyncable(p, syncOpts)), + modified: manifest.modified.filter(p => inScope(p) && !excluded(p) && isSyncable(p, syncOpts)), deleted: unique([ - ...manifest.deleted.filter(p => isSyncable(p, syncOpts)), + ...manifest.deleted.filter(p => inScope(p) && isSyncable(p, syncOpts)), ...renamedToUnsyncable, ]), - renamed: manifest.renamed.filter(r => isSyncable(r.to, syncOpts)), + renamed: manifest.renamed.filter(r => inScope(r.to) && !excluded(r.to) && isSyncable(r.to, syncOpts)), }; + // NAV-4: warn when --exclude filtered out every candidate change — almost + // always a mistyped pattern, and otherwise indistinguishable from + // "up to date" in the output. + if (opts.exclude && opts.exclude.length > 0) { + const excludeCandidates = [...manifest.added, ...manifest.modified] + .filter(p => inScope(p) && isSyncable(p, syncOpts)); + if (excludeCandidates.length > 0 && excludeCandidates.every(excluded)) { + console.warn( + `[gbrain sync] No files matched after applying ${opts.exclude.length} --exclude pattern(s). ` + + `Check your --exclude flags. Patterns: ${JSON.stringify(opts.exclude)}`, + ); + } + } + // Delete pages that became un-syncable (modified but filtered out). // v0.20.0 Cathedral II SP-5: resolveSlugForPath picks the right slug shape // (markdown vs code) based on the chunker's classifier, so a Rust file that @@ -1890,7 +2003,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise` surface. - const unsyncableModified = manifest.modified.filter(p => !isSyncable(p, syncOpts)); + const unsyncableModified = manifest.modified.filter(p => inScope(p) && !isSyncable(p, syncOpts)); // v0.18.0+ multi-source: scope getPage + deletePage to opts.sourceId so // unsyncable cleanup in source A doesn't accidentally sweep same-slug // pages in sources B/C/D. @@ -1938,7 +2051,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise 0) { try { const { extractLinksForSlugs, extractTimelineForSlugs, stampExtracted } = await import('./extract.ts'); - const linksCreated = await extractLinksForSlugs(engine, repoPath, pagesAffected, extractOpts); - const timelineCreated = await extractTimelineForSlugs(engine, repoPath, pagesAffected, extractOpts); + // #774: pages' source_path is git-root-relative, so extract resolves + // files from gitContextRoot (== repoPath realpath when unscoped). + const linksCreated = await extractLinksForSlugs(engine, gitContextRoot, pagesAffected, extractOpts); + const timelineCreated = await extractTimelineForSlugs(engine, gitContextRoot, pagesAffected, extractOpts); if (linksCreated > 0 || timelineCreated > 0) { slog(` Extracted: ${linksCreated} links, ${timelineCreated} timeline entries`); } @@ -2976,11 +3104,21 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise { - // Dry-run: walk the repo, count syncable files, return without writing. + const { gitContextRoot, syncScopeRoot, anchorPath } = roots; + // Scoped sync → slugs/source_path are git-root-relative (matches the + // incremental path's git-diff paths). Unscoped → undefined (dir-relative, + // the pre-#774 behavior, byte-for-byte). + const slugRoot = syncScopeRoot !== gitContextRoot ? gitContextRoot : undefined; + // Dry-run: walk the scope, count syncable files, return without writing. // Fixes the silent-write-on-dry-run bug where performFullSync called // runImport unconditionally regardless of opts.dryRun. // @@ -2990,11 +3128,14 @@ async function performFullSync( // code --dry-run` always reported zero files even when ~1500 code // files were waiting. if (opts.dryRun) { - const allFiles = collectSyncableFiles(repoPath, { strategy: opts.strategy ?? 'markdown' }); + let allFiles = collectSyncableFiles(syncScopeRoot, { strategy: opts.strategy ?? 'markdown' }); + if (opts.exclude && opts.exclude.length > 0) { + allFiles = allFiles.filter(abs => !matchesAnyGlob(relative(syncScopeRoot, abs), opts.exclude)); + } slog( `Full-sync dry run (strategy=${opts.strategy ?? 'markdown'}): ` + `${allFiles.length} file(s) would be imported ` + - `from ${repoPath} @ ${headCommit.slice(0, 8)}.`, + `from ${syncScopeRoot} @ ${headCommit.slice(0, 8)}.`, ); return { status: 'dry_run', @@ -3017,21 +3158,24 @@ async function performFullSync( // sync and the jobs handler. const FULL_SYNC_LARGE_MARKER = Number.MAX_SAFE_INTEGER; const fullConcurrency = autoConcurrency(engine, FULL_SYNC_LARGE_MARKER, opts.concurrency); - slog(`Running full import of ${repoPath}${fullConcurrency > 1 ? ` (${fullConcurrency} workers)` : ''}...`); + slog(`Running full import of ${syncScopeRoot}${fullConcurrency > 1 ? ` (${fullConcurrency} workers)` : ''}...`); const { runImport } = await import('./import.ts'); - const importArgs = [repoPath]; + const importArgs = [syncScopeRoot]; if (opts.noEmbed) importArgs.push('--no-embed'); if (fullConcurrency > 1) importArgs.push('--workers', String(fullConcurrency)); // v0.31.2: thread strategy through so code-strategy first sync // actually enumerates code files (closes bug 1). // v0.30.x: thread sourceId so performFullSync routes pages to the named // source (incremental path already does this). + // #753/#774: thread exclude (--exclude CLI) + slugRoot (monorepo subdir). const _fullImportT0 = Date.now(); serr(`[gbrain phase] sync.fullsync.import start strategy=${opts.strategy ?? 'markdown'}`); const result = await runImport(engine, importArgs, { commit: headCommit, strategy: opts.strategy, sourceId: opts.sourceId, + exclude: opts.exclude, + slugRoot, // issue #1939: performFullSync owns the failure ledger + bookmark via the // shared gate below; don't let runImport double-record or write its own. managedBookmark: true, @@ -3055,9 +3199,9 @@ async function performFullSync( const advanceFull = async (): Promise => { // Persist sync state so the next sync is incremental. Routed through // writeSyncAnchor so --source pins the right sources row. - await writeSyncAnchor(engine, opts.sourceId, 'last_commit', headCommit, newestCommitMs(repoPath)); + await writeSyncAnchor(engine, opts.sourceId, 'last_commit', headCommit, newestCommitMs(gitContextRoot)); await engine.setConfig('sync.last_run', new Date().toISOString()); - await writeSyncAnchor(engine, opts.sourceId, 'repo_path', repoPath); + await writeSyncAnchor(engine, opts.sourceId, 'repo_path', anchorPath); await writeChunkerVersion(engine, opts.sourceId, String(CHUNKER_VERSION)); }; @@ -3084,7 +3228,7 @@ async function performFullSync( ); } await engine.setConfig('sync.last_run', new Date().toISOString()); - await writeSyncAnchor(engine, opts.sourceId, 'repo_path', repoPath); + await writeSyncAnchor(engine, opts.sourceId, 'repo_path', anchorPath); return { status: 'blocked_by_failures', fromCommit: null, @@ -3140,16 +3284,24 @@ async function performFullSync( // backslash paths while a stored source_path can hold git-derived forward // slashes; without normalization every file-backed page mismatches, looks // stale, and the reconcile wipes the whole source. - const currentFiles = collectSyncableFiles(repoPath, { strategy: opts.strategy ?? 'markdown' }) - .map(abs => relative(repoPath, abs)); + // #774: scoped syncs store git-root-relative source_paths (slugRoot), so + // relativize the walk to the same base — otherwise every page mismatches + // and the mass-delete valve trips on a perfectly healthy scoped source. + const currentFiles = collectSyncableFiles(syncScopeRoot, { strategy: opts.strategy ?? 'markdown' }) + .map(abs => relative(slugRoot ?? syncScopeRoot, abs)); const rows = await engine.executeRaw<{ slug: string; source_path: string | null }>( `SELECT slug, source_path FROM pages WHERE source_id = $1 AND source_path IS NOT NULL AND deleted_at IS NULL`, [sid], ); + // #774: a scoped full sync is authoritative ONLY for its scope — pages + // whose source_path lives outside the subpath (e.g. from an earlier + // root-level sync of this source) are out of this walk's sight and must + // not be treated as stale. + const scopePrefix = slugRoot ? relative(gitContextRoot, syncScopeRoot) + '/' : ''; const plan = planReconcileDeletes( rows, currentFiles, - p => isSyncable(p, reconcileSyncOpts), + p => (scopePrefix === '' || p.startsWith(scopePrefix)) && isSyncable(p, reconcileSyncOpts), ); if (plan.staleSlugs.length > 0 && plan.massDelete && !massReconcileAllowed()) { // #2828 mass-delete safety valve: a reconcile that would sweep more than @@ -3400,6 +3552,19 @@ export function composeAbortSignals( return AbortSignal.any(live); } +/** + * #753/#774: `.gitignore` must be managed at the git ROOT — when a source's + * local_path (or --repo) points at a monorepo subdirectory, writing ignore + * entries into the subdir would create a stray `.gitignore` git doesn't + * consult for the repo-level db_only rules. Best-effort: falls back to the + * given path when git discovery fails (manageGitignore no-ops on non-repos). + */ +function manageGitignoreAtGitRoot(path: string, engineKind?: 'pglite' | 'postgres'): void { + let root = path; + try { root = discoverGitRoot(path); } catch { /* best-effort */ } + manageGitignore(root, engineKind); +} + export async function runSync(engine: BrainEngine, args: string[]) { // v0.40 Federated Sync v2: `gbrain sync trigger` subcommand // Routes to runSyncTrigger which queues a 'sync' minion job with @@ -3432,6 +3597,13 @@ Options: --repo Path to the brain repo. Defaults to the path saved by 'gbrain init'. --full Force a full re-sync (rare; usually incremental). + --src-subpath Sync only this subdirectory of the git repo (monorepo + pattern: N logical sources in one repo). Git pull/diff + run at the repo root; imports are scoped to the subdir + and slugs stay root-relative (wiki/page1). Passing the + subdirectory directly as --repo also works. + --exclude Exclude files matching the glob from sync (repeatable; + matched against the scope-relative path). --dry-run Show what would be synced without writing. --skip-failed Acknowledge previously-recorded sync failures so the bookmark can advance past unparseable files. @@ -3591,6 +3763,20 @@ See also: process.exit(1); } const strategyArg = args.find((a, i) => args[i - 1] === '--strategy') as SyncOpts['strategy'] | undefined; + // #753/#774: monorepo subdir-source flags. --exclude is repeatable. + const srcSubpath = args.find((a, i) => args[i - 1] === '--src-subpath') || undefined; + const excludePatterns: string[] = []; + for (let i = 0; i < args.length; i++) { + if (args[i] === '--exclude' && i + 1 < args.length) excludePatterns.push(args[i + 1]); + } + if (syncAll && (srcSubpath || excludePatterns.length > 0)) { + console.error( + `--src-subpath/--exclude scope a single sync invocation; they cannot be combined with --all. ` + + `For --all runs, register the subdirectory as the source's local_path instead ` + + `(gbrain sources add --path /).`, + ); + process.exit(1); + } const concurrencyStr = args.find((a, i) => args[i - 1] === '--concurrency' || args[i - 1] === '--workers'); const parallelStr = args.find((a, i) => args[i - 1] === '--parallel'); // v0.22.13 (PR #490 Q2): parseWorkers throws on '0', '-3', 'foo', '1.5' instead @@ -3850,7 +4036,7 @@ See also: result.status !== 'blocked_by_failures' && result.status !== 'partial' ) { - manageGitignore(src.local_path!, engine.kind); + manageGitignoreAtGitRoot(src.local_path!, engine.kind); } // D18: auto-enqueue embed-backfill per source (unless opted out). // v0.41.13.0 (T7 / D-V3-5): partial excluded — the next clean sync @@ -4038,6 +4224,8 @@ See also: const opts: SyncOpts = { repoPath, dryRun, full, noPull, noEmbed, noExtract, skipFailed, retryFailed, noSchemaPack, sourceId, strategy: strategyArg, concurrency, + srcSubpath, + exclude: excludePatterns.length > 0 ? excludePatterns : undefined, signal: composeAbortSignals(singleSourceInterrupt.signal, singleSourceController?.signal), }; @@ -4121,7 +4309,7 @@ See also: ) { const effectiveRepoPath = opts.repoPath ?? (await getDefaultSourcePath(engine)); if (effectiveRepoPath) { - manageGitignore(effectiveRepoPath, engine.kind); + manageGitignoreAtGitRoot(effectiveRepoPath, engine.kind); } } // v0.42.42.0 (#2139, Step 4b): the inline gate auto-deferred this run's @@ -4170,7 +4358,7 @@ See also: ) { const effectiveRepoPath = opts.repoPath ?? (await getDefaultSourcePath(engine)); if (effectiveRepoPath) { - manageGitignore(effectiveRepoPath, engine.kind); + manageGitignoreAtGitRoot(effectiveRepoPath, engine.kind); } } } catch (e: unknown) { diff --git a/src/core/sync.ts b/src/core/sync.ts index a0d4856f0..455607a6e 100644 --- a/src/core/sync.ts +++ b/src/core/sync.ts @@ -219,7 +219,7 @@ function globToRegex(pattern: string): RegExp { return new RegExp(regex); } -function matchesAnyGlob(path: string, patterns?: string[]): boolean { +export function matchesAnyGlob(path: string, patterns?: string[]): boolean { if (!patterns || patterns.length === 0) return false; const normalized = path.replace(/\\/g, '/'); return patterns.some((pattern) => globToRegex(pattern).test(normalized)); diff --git a/test/sync-monorepo.test.ts b/test/sync-monorepo.test.ts new file mode 100644 index 000000000..3e1f2331a --- /dev/null +++ b/test/sync-monorepo.test.ts @@ -0,0 +1,385 @@ +/** + * #753/#774 — --src-subpath + --exclude monorepo subdir-source support. + * + * A single git repo can hold N logical sources at subdirectories (wiki/, + * memory/, ...). `gbrain sync --src-subpath wiki` (or passing the subdir + * directly as the repo path) scopes file walking + imports to the subdir + * while git operations (pull, rev-parse, diff) run at the discovered repo + * root. Slugs stay git-root-relative (`wiki/page1`) so full and incremental + * syncs of the same scope agree. + * + * Security pins (the point of the feature's guards): + * NAV-1/NAV-2 — `--src-subpath ../escape` and a symlinked subdir pointing + * outside the repo are realpath-checked and rejected before any git op. + * NAV-1 TOCTOU — per-file realpath checks during the incremental import + * drain (see the isPathSafe guard in sync.ts's importOnePath). + * NAV-4 — an --exclude set that filters out everything warns loudly. + * + * Regression note: against pre-#774 master every subdir test fails with + * "Not a git repository". + */ + +import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach } from 'bun:test'; +import { mkdtempSync, writeFileSync, rmSync, mkdirSync, symlinkSync } from 'fs'; +import { join } from 'path'; +import { execSync } from 'child_process'; +import { tmpdir } from 'os'; +import { PGLiteEngine } from '../src/core/pglite-engine.ts'; +import { resetPgliteState } from './helpers/reset-pglite.ts'; + +// Helper: create a minimal valid markdown file +function mdPage(title: string, body = 'Content.'): string { + return `---\ntype: note\ntitle: ${title}\n---\n\n${body}`; +} + +// Helper: init a git repo with author identity +function gitInit(dir: string): void { + execSync('git init', { cwd: dir, stdio: 'pipe' }); + execSync('git config user.email "test@test.com"', { cwd: dir, stdio: 'pipe' }); + execSync('git config user.name "Test"', { cwd: dir, stdio: 'pipe' }); +} + +// Helper: stage + commit everything in a git repo +function gitCommit(dir: string, msg = 'initial'): void { + execSync('git add -A', { cwd: dir, stdio: 'pipe' }); + execSync(`git commit -m "${msg}"`, { cwd: dir, stdio: 'pipe' }); +} + +describe('sync monorepo subdir-source support (#753/#774)', () => { + let engine: PGLiteEngine; + let repoPath: string; + + beforeAll(async () => { + engine = new PGLiteEngine(); + await engine.connect({}); + await engine.initSchema(); + }, 60_000); + + afterAll(async () => { + await engine.disconnect(); + }, 60_000); + + beforeEach(async () => { + await resetPgliteState(engine); + repoPath = mkdtempSync(join(tmpdir(), 'gbrain-monorepo-')); + gitInit(repoPath); + mkdirSync(join(repoPath, 'wiki'), { recursive: true }); + mkdirSync(join(repoPath, 'memory'), { recursive: true }); + writeFileSync(join(repoPath, 'wiki', 'page1.md'), mdPage('Wiki Page 1')); + writeFileSync(join(repoPath, 'wiki', 'page2.md'), mdPage('Wiki Page 2')); + writeFileSync(join(repoPath, 'memory', 'note1.md'), mdPage('Memory Note 1')); + writeFileSync(join(repoPath, 'memory', 'note2.md'), mdPage('Memory Note 2')); + gitCommit(repoPath); + }); + + afterEach(() => { + if (repoPath) rmSync(repoPath, { recursive: true, force: true }); + }); + + // ───────────────────────────────────────────────────────────────────────── + // Back-compat: sync at git root (no srcSubpath) still works + // ───────────────────────────────────────────────────────────────────────── + + test('back-compat: sync at git root without srcSubpath imports all files', async () => { + const { performSync } = await import('../src/commands/sync.ts'); + const result = await performSync(engine, { + repoPath, + noPull: true, + noEmbed: true, + full: true, + }); + expect(result.status).toBe('first_sync'); + expect(result.added).toBe(4); // wiki/page1 + wiki/page2 + memory/note1 + memory/note2 + // Slug shape unchanged for git-root syncs. + expect(await engine.getPage('wiki/page1')).not.toBeNull(); + }); + + // ───────────────────────────────────────────────────────────────────────── + // Auto-discovery: repoPath IS a non-git-root subdir + // ───────────────────────────────────────────────────────────────────────── + + test('auto-discovery: repoPath is a git subdir — discoverGitRoot succeeds', async () => { + const { performSync } = await import('../src/commands/sync.ts'); + // Pass the wiki/ subdir directly as repoPath (no explicit srcSubpath). + // Pre-#774: throws "Not a git repository". + // Post-#774: gitContextRoot = repo root, syncScopeRoot = wiki/. + const result = await performSync(engine, { + repoPath: join(repoPath, 'wiki'), + noPull: true, + noEmbed: true, + full: true, + }); + expect(result.status).toBe('first_sync'); + expect(result.added).toBe(2); // only wiki/page1 + wiki/page2 + // Slugs are git-root-relative in BOTH spellings (subdir repoPath and + // --src-subpath) so full and incremental syncs of the same scope agree. + expect(await engine.getPage('wiki/page1')).not.toBeNull(); + expect(await engine.getPage('page1')).toBeNull(); + }); + + // ───────────────────────────────────────────────────────────────────────── + // srcSubpath explicit flag: scope to subdir from git root + // ───────────────────────────────────────────────────────────────────────── + + test('--src-subpath wiki: only wiki/ files are imported', async () => { + const { performSync } = await import('../src/commands/sync.ts'); + const result = await performSync(engine, { + repoPath, + srcSubpath: 'wiki', + noPull: true, + noEmbed: true, + full: true, + }); + expect(result.status).toBe('first_sync'); + expect(result.added).toBe(2); + // Verify the imported slugs are from wiki/ only (git-root-relative) + const wikiPage = await engine.getPage('wiki/page1'); + expect(wikiPage).not.toBeNull(); + const memoryPage = await engine.getPage('memory/note1'); + expect(memoryPage).toBeNull(); + }); + + test('--src-subpath memory: only memory/ files are imported', async () => { + const { performSync } = await import('../src/commands/sync.ts'); + const result = await performSync(engine, { + repoPath, + srcSubpath: 'memory', + noPull: true, + noEmbed: true, + full: true, + }); + expect(result.status).toBe('first_sync'); + expect(result.added).toBe(2); + const memoryPage = await engine.getPage('memory/note1'); + expect(memoryPage).not.toBeNull(); + const wikiPage = await engine.getPage('wiki/page1'); + expect(wikiPage).toBeNull(); + }); + + // ───────────────────────────────────────────────────────────────────────── + // Two sources in one repo, scoped independently + // ───────────────────────────────────────────────────────────────────────── + + test('2 sources in 1 repo: sync each scope independently, no cross-contamination', async () => { + const { performSync } = await import('../src/commands/sync.ts'); + + const wikiResult = await performSync(engine, { + repoPath, + srcSubpath: 'wiki', + noPull: true, + noEmbed: true, + full: true, + }); + expect(wikiResult.status).toBe('first_sync'); + expect(wikiResult.added).toBe(2); + + // Reset only page state, keep the engine connected for second sync + await resetPgliteState(engine); + + const memResult = await performSync(engine, { + repoPath, + srcSubpath: 'memory', + noPull: true, + noEmbed: true, + full: true, + }); + expect(memResult.status).toBe('first_sync'); + expect(memResult.added).toBe(2); + + // After memory sync, memory pages exist and wiki pages don't + expect(await engine.getPage('memory/note1')).not.toBeNull(); + expect(await engine.getPage('wiki/page1')).toBeNull(); + }); + + // ───────────────────────────────────────────────────────────────────────── + // Incremental sync respects the scope (the gap #774 left untested) + // ───────────────────────────────────────────────────────────────────────── + + test('incremental --src-subpath: only in-scope diff paths are processed', async () => { + const { performSync } = await import('../src/commands/sync.ts'); + + const first = await performSync(engine, { + repoPath, + srcSubpath: 'wiki', + noPull: true, + noEmbed: true, + full: true, + }); + expect(first.status).toBe('first_sync'); + + // Commit 2: touch one file in each scope + add one wiki file. + writeFileSync(join(repoPath, 'wiki', 'page1.md'), mdPage('Wiki Page 1', 'Updated.')); + writeFileSync(join(repoPath, 'memory', 'note1.md'), mdPage('Memory Note 1', 'Updated.')); + writeFileSync(join(repoPath, 'wiki', 'page3.md'), mdPage('Wiki Page 3')); + gitCommit(repoPath, 'second'); + + const second = await performSync(engine, { + repoPath, + srcSubpath: 'wiki', + noPull: true, + noEmbed: true, + }); + expect(second.status).toBe('synced'); + expect(second.added).toBe(1); // wiki/page3 only — memory change filtered by scope + expect(second.modified).toBe(1); // wiki/page1 + expect(await engine.getPage('wiki/page3')).not.toBeNull(); + expect(await engine.getPage('memory/note1')).toBeNull(); + }); + + // ───────────────────────────────────────────────────────────────────────── + // Path-traversal sanitization (NAV-1 + NAV-2) + // ───────────────────────────────────────────────────────────────────────── + + test('path-traversal: --src-subpath ../escape is rejected before any git op', async () => { + const outsideDir = mkdtempSync(join(tmpdir(), 'gbrain-escape-')); + try { + const { performSync } = await import('../src/commands/sync.ts'); + await expect( + performSync(engine, { + repoPath, + srcSubpath: '../' + outsideDir.split('/').pop(), + noPull: true, + noEmbed: true, + full: true, + }), + ).rejects.toThrow(/outside git repo|does not exist/i); + } finally { + rmSync(outsideDir, { recursive: true, force: true }); + } + }); + + test('path-traversal: symlink subdir pointing outside repo is rejected (NAV-1 TOCTOU)', async () => { + const outsideDir = mkdtempSync(join(tmpdir(), 'gbrain-sym-target-')); + writeFileSync(join(outsideDir, 'secret.md'), mdPage('Secret')); + const symlinkPath = join(repoPath, 'symlink-escape'); + try { + symlinkSync(outsideDir, symlinkPath); + const { performSync } = await import('../src/commands/sync.ts'); + await expect( + performSync(engine, { + repoPath, + srcSubpath: 'symlink-escape', + noPull: true, + noEmbed: true, + full: true, + }), + ).rejects.toThrow(/outside git repo/i); + } finally { + rmSync(outsideDir, { recursive: true, force: true }); + } + }); + + test('path-traversal: absolute --src-subpath outside the repo is rejected', async () => { + const outsideDir = mkdtempSync(join(tmpdir(), 'gbrain-abs-escape-')); + writeFileSync(join(outsideDir, 'secret.md'), mdPage('Secret')); + try { + const { performSync } = await import('../src/commands/sync.ts'); + // path.join(repoPath, '/abs/path') keeps the traversal relative, but a + // crafted subpath can still resolve outside via ..-segments; both are + // caught by the same realpath containment check. + await expect( + performSync(engine, { + repoPath, + srcSubpath: join('..', '..', outsideDir.slice(1)), + noPull: true, + noEmbed: true, + full: true, + }), + ).rejects.toThrow(/outside git repo|does not exist/i); + } finally { + rmSync(outsideDir, { recursive: true, force: true }); + } + }); + + // ───────────────────────────────────────────────────────────────────────── + // --exclude: repeatable glob pattern flag + // ───────────────────────────────────────────────────────────────────────── + + test('--exclude: single pattern excludes matching files from full sync', async () => { + const { performSync } = await import('../src/commands/sync.ts'); + // Sync wiki/ but exclude page2.md (patterns are scope-relative) + const result = await performSync(engine, { + repoPath, + srcSubpath: 'wiki', + exclude: ['page2.md'], + noPull: true, + noEmbed: true, + full: true, + }); + expect(result.status).toBe('first_sync'); + expect(result.added).toBe(1); // only page1 (page2 excluded) + expect(await engine.getPage('wiki/page1')).not.toBeNull(); + expect(await engine.getPage('wiki/page2')).toBeNull(); + }); + + test('--exclude: glob pattern with wildcard', async () => { + const { performSync } = await import('../src/commands/sync.ts'); + // Exclude all files matching *2.md + const result = await performSync(engine, { + repoPath, + srcSubpath: 'wiki', + exclude: ['*2.md'], + noPull: true, + noEmbed: true, + full: true, + }); + expect(result.status).toBe('first_sync'); + expect(result.added).toBe(1); // only page1 (page2 excluded by *2.md) + }); + + test('--exclude applies to the incremental path too', async () => { + const { performSync } = await import('../src/commands/sync.ts'); + const first = await performSync(engine, { + repoPath, + srcSubpath: 'wiki', + noPull: true, + noEmbed: true, + full: true, + }); + expect(first.status).toBe('first_sync'); + + writeFileSync(join(repoPath, 'wiki', 'draft-a.md'), mdPage('Draft A')); + writeFileSync(join(repoPath, 'wiki', 'page3.md'), mdPage('Wiki Page 3')); + gitCommit(repoPath, 'drafts'); + + const second = await performSync(engine, { + repoPath, + srcSubpath: 'wiki', + exclude: ['draft-*.md'], + noPull: true, + noEmbed: true, + }); + expect(second.status).toBe('synced'); + expect(second.added).toBe(1); // page3 only; draft-a excluded + expect(await engine.getPage('wiki/page3')).not.toBeNull(); + expect(await engine.getPage('wiki/draft-a')).toBeNull(); + }); + + // ───────────────────────────────────────────────────────────────────────── + // --exclude '**/*' emits warning (NAV-4) + // ───────────────────────────────────────────────────────────────────────── + + test('--exclude **/* emits warning when all files are excluded (NAV-4)', async () => { + const { performSync } = await import('../src/commands/sync.ts'); + const warnMessages: string[] = []; + const origWarn = console.warn; + console.warn = (...args: unknown[]) => { + warnMessages.push(args.join(' ')); + origWarn(...args); + }; + try { + await performSync(engine, { + repoPath, + srcSubpath: 'wiki', + exclude: ['**/*'], + noPull: true, + noEmbed: true, + full: true, + }); + } finally { + console.warn = origWarn; + } + const hasExcludeWarn = warnMessages.some(m => m.includes('--exclude') || m.includes('No files matched')); + expect(hasExcludeWarn).toBe(true); + }); +});