From 10079efe40abd92a3c494b82473e4dcc61789fdb Mon Sep 17 00:00:00 2001 From: Robert - Long Street Labs <39202103+Lazydayz137@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:16:36 -0400 Subject: [PATCH] =?UTF-8?q?feat(sync):=20--missing-path=20skip=20=E2=80=94?= =?UTF-8?q?=20classify=20absent-local=5Fpath=20sources=20in=20--all=20inst?= =?UTF-8?q?ead=20of=20failing=20(#3426)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sources.local_path is machine-specific state in a brain-wide table. Any brain whose sources were registered from more than one machine — or a sanctioned setup mid-migration (topologies.md Topology 2, or the system-of-record git flow before every repo is cloned) — has sources whose checkout is not present on the machine running sync --all. Each surfaced as a hard failure and forced rc=1 every run; on one observed fleet that was 12 phantom failures per hour, training operators to ignore the exit code. --missing-path skip classifies them honestly: ⊘ in the human aggregate, status skipped_missing_path + local_path in the --json envelope, new skipped_count, excluded from error_count and the rc=1 gate. Using the flag outside --all warns instead of silently no-oping. Default stays fail: on a single-machine brain a missing local_path usually means an unmounted volume or deleted checkout, and silently skipping would hide data loss. Skip is explicit opt-in. Pure helpers (parseMissingPathMode, partitionMissingPathSources) exported and unit-tested in the sync-all-parallel style — no DB, no fs. Docs: sync --help, docs/TESTING.md inventory, KEY_FILES.md sync entry. CHANGELOG/VERSION deliberately untouched per the release process. Co-authored-by: Ziggy Co-authored-by: Garry Tan Co-authored-by: Lazydayz137 Co-authored-by: Claude Fable 5 --- docs/TESTING.md | 1 + docs/architecture/KEY_FILES.md | 2 +- src/cli.ts | 2 + src/commands/sync.ts | 143 ++++++++++++++++++++++++++--- test/sync-all-missing-path.test.ts | 111 ++++++++++++++++++++++ 5 files changed, 245 insertions(+), 14 deletions(-) create mode 100644 test/sync-all-missing-path.test.ts diff --git a/docs/TESTING.md b/docs/TESTING.md index 9062dce92..a82bc12cb 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -192,6 +192,7 @@ Unit tests and what they cover: - `test/sync-pull-failed-anchor.serial.test.ts` — #3068 regression: a failed internal `git pull` (local-path origin vs `protocol.file.allow=never`) with zero imports returns `partial`/`pull_failed` (not `up_to_date`), freezes `last_commit` + `last_sync_at`, recovers after a manual pull; fall-through import of local commits preserved. Serial: pins `GBRAIN_HOME` to a temp dir for the whole file. - `test/sync-concurrency.test.ts` — `autoConcurrency()` thresholds + PGLite-forces-serial + explicit-override clamping; `shouldRunParallel()` explicit-bypasses-floor contract; `parseWorkers()` validation rejecting `'0'`/`'-3'`/`'foo'`/`'1.5'`/trailing chars. - `test/sync-parallel.test.ts` — PGLite-routed coverage of the bookmark gate under concurrency, head-drift gate, vanished-file failure capture, PGLite-stays-serial, and the `gbrain-sync` writer-lock contract. +- `test/sync-all-missing-path.test.ts` — `sync --all --missing-path ` pure helpers: `parseMissingPathMode` (default fail, explicit values, loud rejection of bad/dangling values, never swallows a following flag) and `partitionMissingPathSources` (classification driven only by the injected pathExists predicate — no fs; null `local_path` passes through runnable; order preserved). - `test/sync-failures.test.ts` — `classifyErrorCode` regex coverage for all 12 codes against literal production message strings from `markdown.ts` and `import-file.ts`; `summarizeFailuresByCode` sort + pre-classified-honor; `recordSyncFailures` code-field persistence; `acknowledgeSyncFailures` `AcknowledgeResult` shape + backfill on legacy entries. - `test/doctor.test.ts` — doctor command; assertions that `jsonb_integrity` scans the four JSONB write sites and `markdown_body_completeness` is present. - `test/utils.test.ts` — shared SQL utilities + `tryParseEmbedding` null-return and single-warn semantics. diff --git a/docs/architecture/KEY_FILES.md b/docs/architecture/KEY_FILES.md index a7924bcf6..ac0b675cf 100644 --- a/docs/architecture/KEY_FILES.md +++ b/docs/architecture/KEY_FILES.md @@ -310,7 +310,7 @@ per-release `**vX.Y.Z:**` narration — CI enforces this - `src/core/embedding-dim-check.ts` extension — facts.embedding dim drift surface. `readFactsEmbeddingDim(engine): Promise` covers both `vector(N)` and `halfvec(N)` shapes (migration v40 falls back to `vector` on pgvector < 0.7); regex ordering is halfvec-before-vector (substring "vec" appears in "halfvec"; naive `/vector/i` would shadow). `buildFactsAlterRecipe(dims, configured, type)` emits the paste-ready `DROP INDEX IF EXISTS idx_facts_embedding_hnsw; ALTER TABLE facts ALTER COLUMN embedding TYPE halfvec(N) USING embedding::halfvec(N); CREATE INDEX idx_facts_embedding_hnsw ON facts USING hnsw (embedding halfvec_cosine_ops) WHERE ...` flow (NOT bare REINDEX, which doesn't rewrite the index after a column-type change). `assertFactsEmbeddingDimMatchesConfig(engine)` is the preflight — throws `FactsEmbeddingDimMismatchError` (tagged `tag: 'FACTS_EMBEDDING_DIM_MISMATCH'` for parity with the worker-pool MUST_ABORT semantics) when configured dim ≠ column width; cached per-engine via `WeakMap`; PGLite engines silently skip. Doctor check `facts_embedding_width_consistency` (registered after `embedding_width_consistency`) reuses the same helpers with an identical ALTER recipe. Pinned by `test/embedding-dim-check-facts.test.ts`. - `src/core/postgres-engine.ts` extension — `insertFact` + `insertFacts` no longer hardcode `tx.unsafe(\`'${embedLit}'::vector\`)` for the embedding column. `resolveFactsEmbeddingCast()` (private) probes `pg_attribute` once per engine instance (cached in `_factsEmbeddingCastSuffix`) and returns `'::halfvec'` when migration v40 created the column as halfvec, else `'::vector'`; both insert paths use the cached suffix so the cast matches the actual column type (works on older pgvector that lacks implicit auto-cast). Test seam `__resetFactsEmbeddingCastCacheForTest()` clears the per-engine cache. - `src/core/cycle.ts` + `src/core/cycle/extract-atoms.ts` + `src/core/cycle/synthesize-concepts.ts` + `src/commands/extract.ts` + `src/commands/doctor.ts` + `src/core/op-checkpoint.ts` extensions — six daily-driver ops fixes. (1) Batch idempotency: `atomsExistingForHashes(engine, sourceId, hashes[])` (exported from `src/core/cycle/extract-atoms.ts`) replaces the per-hash loop (7K individual queries) with one batched SQL roundtrip returning already-extracted `content_hash16` values; fail-open (SQL error → empty set, extraction proceeds); powered by migration v104 `pages_atom_source_hash_idx` (partial expression index on `frontmatter->>'source_hash'` for atom rows where `deleted_at IS NULL`; Postgres `CREATE INDEX CONCURRENTLY` with invalid-remnant pre-drop, PGLite plain). (2) Shorter cycle lock TTL + active in-phase refresh: `LOCK_TTL_MINUTES = 5` (was 30); `buildYieldDuringPhase(lock, outer)` (exported, with `LockHandle`) calls `lock.refresh()` + any external hook on every fire, throttled to 30s via `maybeYield`, firing both in the main loop AND immediately after every `await chat(...)`; `synthesize_concepts` uses the same throttled hook. A crashed cycle releases its lock 6x faster while a healthy long-running cycle keeps it alive (residual: a single `await chat()` past 5 min can expire the lock mid-await — TODO-OPS-2). (3) Progress wiring: `progress?: ProgressReporter` opt on `ExtractAtomsOpts` and `SynthesizeConceptsOpts`; cycle.ts passes its phase-level reporter down (NOT a child reporter, which would collide on `cycle.extract_atoms.extract_atoms.work`); phases only call `tick()`/`heartbeat()`, cycle.ts owns `start()`/`finish()`. (4) `by-mention` resume: `mentionsFingerprint({source, type, since, gazetteerHash})` in `src/core/op-checkpoint.ts` — the gazetteer hash is load-bearing (adding entity pages mid-pause shifts the hash → new fingerprint → fresh scan against the new gazetteer, never silent skip); `gbrain extract links --by-mention` resumes via `op_checkpoints` with `flushAndCheckpoint` ordering (links flush to DB FIRST, page keys commit to checkpoint SECOND, persist THIRD, so a crash mid-batch leaves the page un-checkpointed and resume re-scans it); persist every 1000 items OR 30s; clean exit clears the checkpoint; `--dry-run` skips both load and write. (5) `sync_consolidation` doctor check (multi-source brains see a paste-ready `gbrain sync --all --parallel 4 --workers 4 --skip-failed`; single-source "not applicable"; SQL errors return `warn` via the check's own try/catch). (6) Test-isolation: `test/cycle-last-full-cycle-at.test.ts` + `test/schema-cli.test.ts` use per-test `GBRAIN_HOME=tempdir`. Pinned by `test/cycle/extract-atoms-batch.test.ts`, `test/cycle/cycle-lock-ttl.test.ts` (regression pin on `LOCK_TTL_MINUTES === 5`), `test/op-checkpoint-mentions-fingerprint.test.ts`, `test/cycle/extract-atoms-progress.test.ts`, `test/cycle/synthesize-concepts-progress.test.ts`, `test/cycle/yield-during-phase-refresh.test.ts`, `test/cycle/yield-during-phase-throttle.test.ts`, `test/extract-by-mention-resume.test.ts`, `test/doctor-sync-consolidation.test.ts`. Companion `sync --all` recipe block in `skills/cron-scheduler/SKILL.md`. `synthesize_concepts` writes concept pages through `importFromContent` (#2163: the same parse→chunk→embed pipeline put_page uses, with put_page's `isAvailable('embedding')` → `noEmbed` gate) so `concepts/` pages carry `content_chunks` + embeddings and are reachable by retrieval (where `source-boost.ts` weights them 1.3×). -- `src/commands/sync.ts` — `gbrain sync` CLI + the `performSync` / `performFullSync` library entrypoints (consumed by the autopilot cycle and the Minion sync handler). `performSync` runs under a writer lock: per-source `gbrain-sync:` whenever `opts.sourceId` is set, wrapped in `withRefreshingLock` from `src/core/db-lock.ts` so long-running sources (250K+ chunks) don't lose the lock at the 30-min TTL mid-run; the bare no-source path uses the SAME refreshing lock (#1794 — it was previously a non-refreshing `tryAcquireDbLock`, stealable mid-run during an incident); `SyncOpts.lockId?: string` is the explicit override. The lock refresh AND its health probe route through the DIRECT session pool so Supavisor transaction-pool exhaustion (`EMAXCONNSESSION`) can't kill renewal; takeover is heartbeat-aware (it will NOT steal a holder whose `last_refreshed_at` is within `GBRAIN_LOCK_STEAL_GRACE_SECONDS`, defending an alive-but-starved holder); the import loop yields the event loop every `GBRAIN_SYNC_YIELD_EVERY` files (`setTimeout(0)`, not `setImmediate` — Bun starves the timers phase) so the refresh `setInterval` heartbeat fires mid-import. This lock-identity invariant prevents a `sync --all` per-source worker racing `sync --source foo` on the global lock from corrupting the same source. `performSync` throws a typed `SyncLockBusyError` when the writer lock is held; the Minion `sync` handler (`src/commands/jobs.ts`) catches it and marks the job SKIPPED (not failed) so a cron/autopilot tick defers to the holder without polluting crash metrics. `performSyncInner` is RESUMABLE (incremental path): it drains a PINNED target commit (`lastCommit..pin`), banking drained file paths via `appendCompleted` (append-only delta into the `op_checkpoint_paths` child table, migration v115 — one row per path, O(delta) not the old O(N²) full-array rewrite), keyed by `syncFingerprint({sourceId, lastCommit})` from `src/core/op-checkpoint.ts` (paths under `op:'sync'`; the pinned target under `op:'sync-target'`), and advances `last_commit`/`last_sync_at` ONLY at full import completion. Checkpoint writes route through the DIRECT session pool + bounded retry so they survive `EMAXCONNSESSION`; the flush cadence is first-file then every `GBRAIN_SYNC_CHECKPOINT_EVERY` (default 1000) files OR `GBRAIN_SYNC_CHECKPOINT_SECONDS` (default 10s), with a race-safe `pendingCheckpointPaths` delta (single-flight swap, re-merge on failure) under parallel workers; a SIGTERM banks the in-flight delta via a no-retry one-shot (`appendCompletedOnce`, ordered before lock release through `registerCleanup`); and sustained flush failure aborts the run with `reason:'checkpoint_unavailable'` after `GBRAIN_SYNC_MAX_CHECKPOINT_FAILURES` consecutive fails rather than importing work it can never bank (every partial/blocked exit logs the banked-file count). A sync killed mid-import banks its progress, leaves the anchor unmoved (the source stays correctly stale to the autopilot scheduler — `last_sync_at` is never bumped on a partial), and the next run `resumeFilter`s the same fixed diff to skip done files. The pin is the checkpoint's stored target when still reachable from HEAD, else live HEAD (a history rewrite / reset re-pins); completion advances to the pin, NOT live HEAD, so commits landing past the pin are a clean next-sync diff — this closes the cross-run staleness window. After import a pin-reachability gate (`git merge-base --is-ancestor pin HEAD`) replaces the old strict head-drift gate: forward commits on top of the pin (e.g. a background enrich process committing to the same repo every ~2 min) are SAFE and no longer block the run; only a real rewrite (pin not an ancestor of HEAD) blocks and discards the checkpoint. A file added in `lastCommit..pin` but gone from disk (deleted by a commit after the pin) is SKIPPED and marked completed, not recorded as a failure. Downstream extract/facts/embed are size-gated: inline only for `totalChanges <= 100`; large syncs defer to the resumable `extract --stale` watermark + `embed --stale`/backfill + the facts cycle phases, so a 44K-page facts/embed pass never re-blocks import convergence (sync convergence == import convergence). Worker engines wrap in try/finally so disconnect always fires; both PGLite-detection sites use `engine.kind === 'pglite'`. CLI accepts `--workers N` (alias `--concurrency N`) validated via `parseWorkers` (explicit bypasses the file-count floor; auto path defers to `autoConcurrency()`). The newest-first descending-lex order uses `sortNewestFirst(addsAndMods)` from `src/core/sort-newest-first.ts` (shared with `gbrain import`). `gbrain sync --all` runs a continuous worker pool: `parseWorkers`-validated `--parallel N` (default `min(sourceCount, --workers, DEFAULT_PARALLEL_SOURCES=4)`), long-lived async workers pulling from a shared FIFO queue (no head-of-line blocking), per-source `withSourcePrefix(src.id, ...)` so every `slog`/`serr` line carries `[]`; `--skip-failed`/`--retry-failed` are scoped per source (`acknowledgeFailures(sourceId)`; `--all` acks every source, single-source acks only its own) and run UNDER parallel — the #1939 failure ledger is per-`(source_id, path)` and serialized through `withLedgerLock`, so the old "not supported under parallel, re-run with --serial" refusal is lifted (#2139, which also removed the forcing-function that pushed recovery syncs to `--serial` and thus armed the inline cost gate); a connection-budget stderr warning fires when `parallel × workers × 2 > 16` (the `× 2 per-file pool` factor: each per-file worker opens its own `PostgresEngine` with `poolSize=2`). Exports `resolveParallelism`, `syncOneSource`, `buildSyncStatusReport`, `printSyncStatusReport`, `SyncStatusReport` back the `gbrain sources status` dashboard. `--json` envelope `{schema_version: 1, sources, parallel, ok_count, error_count}` on stdout; human banners route to stderr via `humanSink` so `jq` parses cleanly. Exit matrix: 0 all ok, 1 any error. (The non-TTY cost gate no longer exits 2 — it auto-defers; #2139.) The dashboard SQL is `content_chunks ch JOIN pages pg ON pg.id = ch.page_id WHERE pg.deleted_at IS NULL` with `archived = false` at the caller; embedding column resolved via `resolveEmbeddingColumn(undefined, cfg)` from `src/core/search/embedding-column.ts` so Voyage/multimodal/non-default-column brains count against the column they use; errors propagate (no swallow-catch). The sync delete loop is interleaved per-batch resolve+delete using `engine.resolveSlugsByPaths` + `engine.deletePages` from `src/core/engine.ts` (73K-delete commit: ~146K SQL round-trips → ~292, closing the cascade-staleness class where one big-delete commit jammed every other source's sync); per-batch try-catch decomposes batch DELETE failures to per-slug `deletePage` fallback, unrecoverable per-slug failures land in `failedFiles`; `pagesAffected` filters to confirmed-deleted slugs. The rename loop gets the same batched slug-resolve; a rename whose destination is un-syncable folds the source path into the delete set so the old page can't orphan (#1970). An entry-time bookmark-reachability guard distinguishes a gc'd anchor (`cat-file` fails → `performFullSync`) from a history-rewrite anchor that is merely no-longer-an-ancestor but still on disk: the latter is diffed tree-to-tree directly (`git diff lastCommit..pin` is an endpoint-tree compare, ancestry not required) so a force-push / `master`→`main` consolidation imports only the real delta instead of re-walking the whole tree forever (#1970); an oversized or failed diff degrades to `performFullSync`. `performFullSync` is itself authoritative for deletes — after an advancing full import it purges file-backed pages (`source_path != null` AND strategy-aware `isSyncable`) whose source file no longer exists, sparing `put_page`/manual pages (null `source_path`) and metafiles. The stale-file decision routes through the pure, exported `planReconcileDeletes(rows, currentFiles, isSyncablePath)`: it normalizes path separators on both sides of the membership test (a Windows `path.relative` backslash path vs a git-derived forward-slash `source_path` would otherwise mark every page stale and wipe the source) and computes a mass-delete signal — when the reconcile would delete more than `MASS_RECONCILE_RATIO` (50%) of the file-backed pages the strategy manages, on a source holding more than `MASS_RECONCILE_MIN_PAGES` (20) of them, the delete is REFUSED with a loud stderr warning (that shape is almost always a path-comparison bug or the wrong repo path, not a genuine bulk deletion); `GBRAIN_ALLOW_MASS_RECONCILE=1` restores the unguarded delete for genuinely intended bulk removals. Pinned by `test/sync-reconcile-mass-delete.test.ts`. Below the valve, stale pages are partitioned by git history via exported `listEverCommittedPaths(repoPath)` (one `git log --all --no-renames --diff-filter=A --name-only` pass; null on non-git dirs → unchanged behavior): a stale path that EVER existed in history was genuinely deleted → reconciled; a path with NO history is DB-only write-through (never committed/pushed, e.g. lost to a fresh clone) → the page is KEPT and its markdown re-exported to the working tree via `writePageThrough`, with a stderr hint to commit it (#2426 — "absent from git" is the symptom of the missing write-through commit, not evidence the content is disposable). Pinned by `test/sync-reconcile-db-only.serial.test.ts`. `resolveSlugByPathOrSourcePath` at `sync.ts:267` delegates to `engine.resolveSlugsByPaths` when `sourceId` is set, keeping legacy `executeRaw` fallback for the no-sourceId path. `failedFiles` is hoisted to the top of `performSyncInner` so both delete-decompose and import loops feed the same bookmark gate. The cost gate is the shared `runInlineCostGate` (one implementation on BOTH the `--all` and single-source paths; runs at the command layer, never inside `performSync`), mode-aware via `willEmbedSynchronously` + posture-aware `shouldBlockSync` from `src/core/embedding.ts` (#2139). The DEFERRED path (v2 on, parallel) is INFORMATIONAL (embedding goes to per-source `embed-backfill` jobs with their own `$X/source/24h` cap, default $25 via `SPEND_CAP_CONFIG_KEY` from `embed-backfill-submit.ts`; prints the cap + backlog + queued-job count, NEVER exits 2). The INLINE path (v2 off, or `--serial`) gates on the DELTA estimate vs `sync.cost_gate_min_usd` (default $0.50): below floor proceeds; above floor in a TTY prompts `[y/N]`; above floor in a non-TTY/`--json` session AUTO-DEFERS embeds to capped backfill jobs and exits 0 (NEVER exit 2 — the wedged-cron fix); `spend.posture=tokenmax` makes it informational and proceeds inline. The estimate MIRRORS EXECUTION instead of pricing the whole tree: `estimateInlineNewTokens` routes through the shared `computeSyncDelta` (`src/core/sync-delta.ts`) — fetch-first against `origin/`, prices only the committed delta (a dirty-but-caught-up tree → $0), with a full-tree CEILING only on the fail-open rungs (chunker drift, first sync, git-unavailable) honestly labeled; `--full` adds the stale backlog (full sync sweeps it inline). Return shape carries `estimateKind: 'delta'|'ceiling'|'mixed'|'unchanged'` + `ceilingReasons`. Helpers `resolveCostGateFloorUsd(engine)` + `resolveBackfillCapUsd(engine)` resolve via `parseUsdLimit` (`off`/`unlimited` → `Infinity`; floor accepts `0` = block-on-any-spend). JSON envelopes carry `mode` + `gate` discriminators (`dry_run | deferred_notice | below_floor | auto_deferred_embeds | posture_tokenmax`) + a paste-ready `hint`; `Infinity` floors/caps render as the string `'unlimited'` (never raw, which JSON-serializes to `null`); `SyncStatusReportSource` gains `backfill_queued`/`backfill_active`/`backfill_last_completed_at`; cost previews read `getEmbeddingModelName()` (no hardcoded OpenAI). Format splits on the explicit `--json` flag only (human text otherwise). `SyncOpts.noSchemaPack` (CLI `--no-schema-pack`, threaded through `performSync` AND `syncOneSource`) skips `loadActivePack` so pages fall back to legacy prefix typing — an escape hatch when a suspect pack regex wedges a sync. A per-file BEGIN heartbeat `if (process.env.GBRAIN_SYNC_TRACE) serr('[sync] begin import: ')` fires BEFORE `importFile` (the `progress.tick` fires only AFTER) so a stuck file is a begin-line with no matching completion. Triage doc: `docs/architecture/serve-sync-concurrency.md` (PGLite single-writer serve↔sync contention + the `GBRAIN_SYNC_TRACE` + `--no-schema-pack` recipes). Pinned by `test/e2e/sync-status-pglite.test.ts` (IRON-RULE: PGLite seeds 2 sources × pages × chunks, soft-deletes 1 page, archives 1 source, validates the SQL excludes both AND uses the active embedding column), `test/sync-cost-gate.serial.test.ts`, `test/sync-cost-preview.test.ts`. Runaway-sync protection: `resolveSyncHardDeadline(args, {isTty, env, defaultNonTtySec?})` resolves a wall-clock hard deadline (precedence `--no-hard-deadline` > `--hard-deadline ` > `--timeout `(non-`--all`, which auto-arms the backstop) > `GBRAIN_SYNC_MAX_RUNTIME_SECONDS` env > non-TTY default 3600s > none; `HARD_DEADLINE_GRACE_SEC=30`). `src/cli.ts` installs the out-of-band watchdog (see `src/core/process-watchdog.ts`) for the sync command BEFORE `connectEngine` and disposes it in the dispatch `finally`, so even an event-loop-starved sync — or a connect-phase hang — is SIGTERM-then-SIGKILLed by the deadline instead of orphaning under cron. `runSync` registers a SIGINT handler that aborts an interrupt `AbortController` composed via `composeAbortSignals(...)` (an `AbortSignal.any` wrapper over the defined signals) with the per-source `--timeout` signal, so Ctrl-C returns a clean `partial` and releases the lock through the normal `finally` (process-cleanup.ts owns SIGTERM lock-release; the watchdog owns the hard kill). `withRefreshingLock` `unref()`s its refresh `setInterval`. The spin's own root cause is not yet pinned (leading lead is catastrophic-backtracking in a pack link-inference regex, bounded by the redos-guard); the watchdog heartbeat plus the existing `[gbrain phase]` breadcrumbs are the diagnosis surface. Pinned by `test/sync-hard-deadline.test.ts` (resolution precedence + `composeAbortSignals`). Monorepo subdir sources (#753/#774): `--src-subpath ` (or a repo path that IS a subdir — auto-discovery via `discoverGitRoot`, i.e. `git rev-parse --show-toplevel`) splits the repo path into `gitContextRoot` (all git ops: pull/diff/rev-parse/cat-file) and `syncScopeRoot` (walk/import/delete/rename scope); scoped syncs use git-root-relative slugs + `source_path` (full sync threads `slugRoot` into `runImport`) so full and incremental agree; NAV-1/NAV-2 realpath containment rejects `../`-traversal and symlinked scopes resolving outside the repo BEFORE any git op, and a per-file realpath guard (`isPathSafe`) refuses symlink-escape files in the incremental drain and rename reimport (fail-closed into `failedFiles`, so the bookmark can't advance past an escape); the full-sync reconcile is scope-restricted so a scoped sync never sweeps out-of-scope pages. `--exclude ` (repeatable) filters scope-relative paths in both full and incremental paths; exclusion never deletes previously-imported pages (conservative, matching the #1433 metafile posture); an all-excluded run warns loudly (NAV-4). A warn-and-continue internal `git pull` failure (non-timeout class — e.g. a local-path origin rejected by `protocol.file.allow=never`) still falls through to sync the local working tree, but a ZERO-import run after a failed pull returns `partial` with `reason: 'pull_failed'` instead of `up_to_date`: `last_commit` AND the `last_sync_at` heartbeat stay frozen (so doctor `sync_freshness` / `sources status` staleness fires), the single-source CLI exits non-zero, `sync --all` exits non-zero if any source hit it (JSON envelope carries the per-source `reason`), and the autopilot cycle's sync phase maps it to `warn`. Timeout-class partials keep their pre-existing exit-0 / phase-`ok` semantics (they converge on retry; a failing pull does not). Pinned by `test/sync-pull-failed-anchor.serial.test.ts`. +- `src/commands/sync.ts` — `gbrain sync` CLI + the `performSync` / `performFullSync` library entrypoints (consumed by the autopilot cycle and the Minion sync handler). `performSync` runs under a writer lock: per-source `gbrain-sync:` whenever `opts.sourceId` is set, wrapped in `withRefreshingLock` from `src/core/db-lock.ts` so long-running sources (250K+ chunks) don't lose the lock at the 30-min TTL mid-run; the bare no-source path uses the SAME refreshing lock (#1794 — it was previously a non-refreshing `tryAcquireDbLock`, stealable mid-run during an incident); `SyncOpts.lockId?: string` is the explicit override. The lock refresh AND its health probe route through the DIRECT session pool so Supavisor transaction-pool exhaustion (`EMAXCONNSESSION`) can't kill renewal; takeover is heartbeat-aware (it will NOT steal a holder whose `last_refreshed_at` is within `GBRAIN_LOCK_STEAL_GRACE_SECONDS`, defending an alive-but-starved holder); the import loop yields the event loop every `GBRAIN_SYNC_YIELD_EVERY` files (`setTimeout(0)`, not `setImmediate` — Bun starves the timers phase) so the refresh `setInterval` heartbeat fires mid-import. This lock-identity invariant prevents a `sync --all` per-source worker racing `sync --source foo` on the global lock from corrupting the same source. `performSync` throws a typed `SyncLockBusyError` when the writer lock is held; the Minion `sync` handler (`src/commands/jobs.ts`) catches it and marks the job SKIPPED (not failed) so a cron/autopilot tick defers to the holder without polluting crash metrics. `performSyncInner` is RESUMABLE (incremental path): it drains a PINNED target commit (`lastCommit..pin`), banking drained file paths via `appendCompleted` (append-only delta into the `op_checkpoint_paths` child table, migration v115 — one row per path, O(delta) not the old O(N²) full-array rewrite), keyed by `syncFingerprint({sourceId, lastCommit})` from `src/core/op-checkpoint.ts` (paths under `op:'sync'`; the pinned target under `op:'sync-target'`), and advances `last_commit`/`last_sync_at` ONLY at full import completion. Checkpoint writes route through the DIRECT session pool + bounded retry so they survive `EMAXCONNSESSION`; the flush cadence is first-file then every `GBRAIN_SYNC_CHECKPOINT_EVERY` (default 1000) files OR `GBRAIN_SYNC_CHECKPOINT_SECONDS` (default 10s), with a race-safe `pendingCheckpointPaths` delta (single-flight swap, re-merge on failure) under parallel workers; a SIGTERM banks the in-flight delta via a no-retry one-shot (`appendCompletedOnce`, ordered before lock release through `registerCleanup`); and sustained flush failure aborts the run with `reason:'checkpoint_unavailable'` after `GBRAIN_SYNC_MAX_CHECKPOINT_FAILURES` consecutive fails rather than importing work it can never bank (every partial/blocked exit logs the banked-file count). A sync killed mid-import banks its progress, leaves the anchor unmoved (the source stays correctly stale to the autopilot scheduler — `last_sync_at` is never bumped on a partial), and the next run `resumeFilter`s the same fixed diff to skip done files. The pin is the checkpoint's stored target when still reachable from HEAD, else live HEAD (a history rewrite / reset re-pins); completion advances to the pin, NOT live HEAD, so commits landing past the pin are a clean next-sync diff — this closes the cross-run staleness window. After import a pin-reachability gate (`git merge-base --is-ancestor pin HEAD`) replaces the old strict head-drift gate: forward commits on top of the pin (e.g. a background enrich process committing to the same repo every ~2 min) are SAFE and no longer block the run; only a real rewrite (pin not an ancestor of HEAD) blocks and discards the checkpoint. A file added in `lastCommit..pin` but gone from disk (deleted by a commit after the pin) is SKIPPED and marked completed, not recorded as a failure. Downstream extract/facts/embed are size-gated: inline only for `totalChanges <= 100`; large syncs defer to the resumable `extract --stale` watermark + `embed --stale`/backfill + the facts cycle phases, so a 44K-page facts/embed pass never re-blocks import convergence (sync convergence == import convergence). Worker engines wrap in try/finally so disconnect always fires; both PGLite-detection sites use `engine.kind === 'pglite'`. CLI accepts `--workers N` (alias `--concurrency N`) validated via `parseWorkers` (explicit bypasses the file-count floor; auto path defers to `autoConcurrency()`). The newest-first descending-lex order uses `sortNewestFirst(addsAndMods)` from `src/core/sort-newest-first.ts` (shared with `gbrain import`). `gbrain sync --all` runs a continuous worker pool: `parseWorkers`-validated `--parallel N` (default `min(sourceCount, --workers, DEFAULT_PARALLEL_SOURCES=4)`), long-lived async workers pulling from a shared FIFO queue (no head-of-line blocking), per-source `withSourcePrefix(src.id, ...)` so every `slog`/`serr` line carries `[]`; `--skip-failed`/`--retry-failed` are scoped per source (`acknowledgeFailures(sourceId)`; `--all` acks every source, single-source acks only its own) and run UNDER parallel — the #1939 failure ledger is per-`(source_id, path)` and serialized through `withLedgerLock`, so the old "not supported under parallel, re-run with --serial" refusal is lifted (#2139, which also removed the forcing-function that pushed recovery syncs to `--serial` and thus armed the inline cost gate); a connection-budget stderr warning fires when `parallel × workers × 2 > 16` (the `× 2 per-file pool` factor: each per-file worker opens its own `PostgresEngine` with `poolSize=2`). Exports `resolveParallelism`, `syncOneSource`, `buildSyncStatusReport`, `printSyncStatusReport`, `SyncStatusReport` back the `gbrain sources status` dashboard. `--json` envelope `{schema_version: 1, sources, parallel, ok_count, error_count, skipped_count}` on stdout; human banners route to stderr via `humanSink` so `jq` parses cleanly. Exit matrix: 0 all ok (sources skipped by `--missing-path skip` count as ok), 1 any error. `--missing-path ` (default fail) handles sources whose `local_path` does not exist on this machine — machine-specific state in a brain-wide table, so a brain registered from several machines fails every foreign source on every run; `skip` classifies them `skipped_missing_path` (⊘ line, envelope entry with `local_path`, excluded from `error_count` and the rc gate) via the exported pure helpers `parseMissingPathMode` + `partitionMissingPathSources`, pinned by `test/sync-all-missing-path.test.ts`; default `fail` stays loud because on a single-machine brain a missing path usually means an unmounted volume. (The non-TTY cost gate no longer exits 2 — it auto-defers; #2139.) The dashboard SQL is `content_chunks ch JOIN pages pg ON pg.id = ch.page_id WHERE pg.deleted_at IS NULL` with `archived = false` at the caller; embedding column resolved via `resolveEmbeddingColumn(undefined, cfg)` from `src/core/search/embedding-column.ts` so Voyage/multimodal/non-default-column brains count against the column they use; errors propagate (no swallow-catch). The sync delete loop is interleaved per-batch resolve+delete using `engine.resolveSlugsByPaths` + `engine.deletePages` from `src/core/engine.ts` (73K-delete commit: ~146K SQL round-trips → ~292, closing the cascade-staleness class where one big-delete commit jammed every other source's sync); per-batch try-catch decomposes batch DELETE failures to per-slug `deletePage` fallback, unrecoverable per-slug failures land in `failedFiles`; `pagesAffected` filters to confirmed-deleted slugs. The rename loop gets the same batched slug-resolve; a rename whose destination is un-syncable folds the source path into the delete set so the old page can't orphan (#1970). An entry-time bookmark-reachability guard distinguishes a gc'd anchor (`cat-file` fails → `performFullSync`) from a history-rewrite anchor that is merely no-longer-an-ancestor but still on disk: the latter is diffed tree-to-tree directly (`git diff lastCommit..pin` is an endpoint-tree compare, ancestry not required) so a force-push / `master`→`main` consolidation imports only the real delta instead of re-walking the whole tree forever (#1970); an oversized or failed diff degrades to `performFullSync`. `performFullSync` is itself authoritative for deletes — after an advancing full import it purges file-backed pages (`source_path != null` AND strategy-aware `isSyncable`) whose source file no longer exists, sparing `put_page`/manual pages (null `source_path`) and metafiles. The stale-file decision routes through the pure, exported `planReconcileDeletes(rows, currentFiles, isSyncablePath)`: it normalizes path separators on both sides of the membership test (a Windows `path.relative` backslash path vs a git-derived forward-slash `source_path` would otherwise mark every page stale and wipe the source) and computes a mass-delete signal — when the reconcile would delete more than `MASS_RECONCILE_RATIO` (50%) of the file-backed pages the strategy manages, on a source holding more than `MASS_RECONCILE_MIN_PAGES` (20) of them, the delete is REFUSED with a loud stderr warning (that shape is almost always a path-comparison bug or the wrong repo path, not a genuine bulk deletion); `GBRAIN_ALLOW_MASS_RECONCILE=1` restores the unguarded delete for genuinely intended bulk removals. Pinned by `test/sync-reconcile-mass-delete.test.ts`. Below the valve, stale pages are partitioned by git history via exported `listEverCommittedPaths(repoPath)` (one `git log --all --no-renames --diff-filter=A --name-only` pass; null on non-git dirs → unchanged behavior): a stale path that EVER existed in history was genuinely deleted → reconciled; a path with NO history is DB-only write-through (never committed/pushed, e.g. lost to a fresh clone) → the page is KEPT and its markdown re-exported to the working tree via `writePageThrough`, with a stderr hint to commit it (#2426 — "absent from git" is the symptom of the missing write-through commit, not evidence the content is disposable). Pinned by `test/sync-reconcile-db-only.serial.test.ts`. `resolveSlugByPathOrSourcePath` at `sync.ts:267` delegates to `engine.resolveSlugsByPaths` when `sourceId` is set, keeping legacy `executeRaw` fallback for the no-sourceId path. `failedFiles` is hoisted to the top of `performSyncInner` so both delete-decompose and import loops feed the same bookmark gate. The cost gate is the shared `runInlineCostGate` (one implementation on BOTH the `--all` and single-source paths; runs at the command layer, never inside `performSync`), mode-aware via `willEmbedSynchronously` + posture-aware `shouldBlockSync` from `src/core/embedding.ts` (#2139). The DEFERRED path (v2 on, parallel) is INFORMATIONAL (embedding goes to per-source `embed-backfill` jobs with their own `$X/source/24h` cap, default $25 via `SPEND_CAP_CONFIG_KEY` from `embed-backfill-submit.ts`; prints the cap + backlog + queued-job count, NEVER exits 2). The INLINE path (v2 off, or `--serial`) gates on the DELTA estimate vs `sync.cost_gate_min_usd` (default $0.50): below floor proceeds; above floor in a TTY prompts `[y/N]`; above floor in a non-TTY/`--json` session AUTO-DEFERS embeds to capped backfill jobs and exits 0 (NEVER exit 2 — the wedged-cron fix); `spend.posture=tokenmax` makes it informational and proceeds inline. The estimate MIRRORS EXECUTION instead of pricing the whole tree: `estimateInlineNewTokens` routes through the shared `computeSyncDelta` (`src/core/sync-delta.ts`) — fetch-first against `origin/`, prices only the committed delta (a dirty-but-caught-up tree → $0), with a full-tree CEILING only on the fail-open rungs (chunker drift, first sync, git-unavailable) honestly labeled; `--full` adds the stale backlog (full sync sweeps it inline). Return shape carries `estimateKind: 'delta'|'ceiling'|'mixed'|'unchanged'` + `ceilingReasons`. Helpers `resolveCostGateFloorUsd(engine)` + `resolveBackfillCapUsd(engine)` resolve via `parseUsdLimit` (`off`/`unlimited` → `Infinity`; floor accepts `0` = block-on-any-spend). JSON envelopes carry `mode` + `gate` discriminators (`dry_run | deferred_notice | below_floor | auto_deferred_embeds | posture_tokenmax`) + a paste-ready `hint`; `Infinity` floors/caps render as the string `'unlimited'` (never raw, which JSON-serializes to `null`); `SyncStatusReportSource` gains `backfill_queued`/`backfill_active`/`backfill_last_completed_at`; cost previews read `getEmbeddingModelName()` (no hardcoded OpenAI). Format splits on the explicit `--json` flag only (human text otherwise). `SyncOpts.noSchemaPack` (CLI `--no-schema-pack`, threaded through `performSync` AND `syncOneSource`) skips `loadActivePack` so pages fall back to legacy prefix typing — an escape hatch when a suspect pack regex wedges a sync. A per-file BEGIN heartbeat `if (process.env.GBRAIN_SYNC_TRACE) serr('[sync] begin import: ')` fires BEFORE `importFile` (the `progress.tick` fires only AFTER) so a stuck file is a begin-line with no matching completion. Triage doc: `docs/architecture/serve-sync-concurrency.md` (PGLite single-writer serve↔sync contention + the `GBRAIN_SYNC_TRACE` + `--no-schema-pack` recipes). Pinned by `test/e2e/sync-status-pglite.test.ts` (IRON-RULE: PGLite seeds 2 sources × pages × chunks, soft-deletes 1 page, archives 1 source, validates the SQL excludes both AND uses the active embedding column), `test/sync-cost-gate.serial.test.ts`, `test/sync-cost-preview.test.ts`. Runaway-sync protection: `resolveSyncHardDeadline(args, {isTty, env, defaultNonTtySec?})` resolves a wall-clock hard deadline (precedence `--no-hard-deadline` > `--hard-deadline ` > `--timeout `(non-`--all`, which auto-arms the backstop) > `GBRAIN_SYNC_MAX_RUNTIME_SECONDS` env > non-TTY default 3600s > none; `HARD_DEADLINE_GRACE_SEC=30`). `src/cli.ts` installs the out-of-band watchdog (see `src/core/process-watchdog.ts`) for the sync command BEFORE `connectEngine` and disposes it in the dispatch `finally`, so even an event-loop-starved sync — or a connect-phase hang — is SIGTERM-then-SIGKILLed by the deadline instead of orphaning under cron. `runSync` registers a SIGINT handler that aborts an interrupt `AbortController` composed via `composeAbortSignals(...)` (an `AbortSignal.any` wrapper over the defined signals) with the per-source `--timeout` signal, so Ctrl-C returns a clean `partial` and releases the lock through the normal `finally` (process-cleanup.ts owns SIGTERM lock-release; the watchdog owns the hard kill). `withRefreshingLock` `unref()`s its refresh `setInterval`. The spin's own root cause is not yet pinned (leading lead is catastrophic-backtracking in a pack link-inference regex, bounded by the redos-guard); the watchdog heartbeat plus the existing `[gbrain phase]` breadcrumbs are the diagnosis surface. Pinned by `test/sync-hard-deadline.test.ts` (resolution precedence + `composeAbortSignals`). Monorepo subdir sources (#753/#774): `--src-subpath ` (or a repo path that IS a subdir — auto-discovery via `discoverGitRoot`, i.e. `git rev-parse --show-toplevel`) splits the repo path into `gitContextRoot` (all git ops: pull/diff/rev-parse/cat-file) and `syncScopeRoot` (walk/import/delete/rename scope); scoped syncs use git-root-relative slugs + `source_path` (full sync threads `slugRoot` into `runImport`) so full and incremental agree; NAV-1/NAV-2 realpath containment rejects `../`-traversal and symlinked scopes resolving outside the repo BEFORE any git op, and a per-file realpath guard (`isPathSafe`) refuses symlink-escape files in the incremental drain and rename reimport (fail-closed into `failedFiles`, so the bookmark can't advance past an escape); the full-sync reconcile is scope-restricted so a scoped sync never sweeps out-of-scope pages. `--exclude ` (repeatable) filters scope-relative paths in both full and incremental paths; exclusion never deletes previously-imported pages (conservative, matching the #1433 metafile posture); an all-excluded run warns loudly (NAV-4). A warn-and-continue internal `git pull` failure (non-timeout class — e.g. a local-path origin rejected by `protocol.file.allow=never`) still falls through to sync the local working tree, but a ZERO-import run after a failed pull returns `partial` with `reason: 'pull_failed'` instead of `up_to_date`: `last_commit` AND the `last_sync_at` heartbeat stay frozen (so doctor `sync_freshness` / `sources status` staleness fires), the single-source CLI exits non-zero, `sync --all` exits non-zero if any source hit it (JSON envelope carries the per-source `reason`), and the autopilot cycle's sync phase maps it to `warn`. Timeout-class partials keep their pre-existing exit-0 / phase-`ok` semantics (they converge on retry; a failing pull does not). Pinned by `test/sync-pull-failed-anchor.serial.test.ts`. - `src/commands/import.ts` — `gbrain import` CLI + `runImport` library entrypoint. Uses a path-set checkpoint via `src/core/import-checkpoint.ts` (the walk still applies `sortNewestFirst()` for embed-cost ordering, but checkpoint correctness no longer depends on sort order). A file enters `completed: Set` only when its `processFile` returns success (including content-hash short-circuit no-ops); failed files never enter the set so the next run retries them automatically with no manual `~/.gbrain/import-checkpoint.json` delete. This closes three classes: parallel-import-with-slow-worker dropping the slow file on crash-resume (the slow file isn't in `completed` until its own `processFile` resolves), failed-file-bumps-counter-past-itself (failures don't add to `completed`), and sort-flip-drops-newest-N-on-cross-version-resume (order is no longer part of the checkpoint). Old positional checkpoints are detected and discarded with a stderr line on first resume (re-walking is cheap because `content_hash` short-circuits unchanged files). Checkpoint persists every 100 successful adds, not every 100 processed files. The `managedBookmark` opt (set by `performFullSync` when `runImport` is the full-sync engine) suppresses `runImport`'s own `sync.last_commit` advance so the shared `applySyncFailureGate` (`src/core/sync-failure-ledger.ts`) owns the bookmark + failure-ledger gating on that path — one gate decides advance/block/auto-skip across both sync paths. Pinned by `test/import-checkpoint.test.ts` + `test/import-resume.test.ts` (incl. the SLUG_MISMATCH retry regression). `collectSyncableFiles`' shared emit filter `isCollectibleForWalker` applies the SAME segment-level `pruneDir` gate as incremental sync's `classifySync` — load-bearing for the `git ls-files` fast path, which enumerates tracked files under dot-dirs/vendored trees that the FS walk never descends into; without it `sync --full` imported (and resurrected soft-deleted) pages incremental sync excludes (#2607). Pinned by `test/import-git-fastpath-prune.test.ts`. `runImport` opts also carry `exclude` (glob filter over dir-relative paths, threaded by `performFullSync` for `sync --exclude`; warns when every file is excluded — NAV-4) and `slugRoot` (slug/`source_path` base for monorepo subdir syncs, #753/#774; the resume checkpoint stays dir-relative per `resumeFilter`'s contract).- `src/core/import-checkpoint.ts` — `loadCheckpoint(brainDir)`, `saveCheckpoint(brainDir, completed)`, `resumeFilter(files, completed, brainDir)`, `clearCheckpoint()`, plus the `ImportCheckpoint` type. Path-set format `{schema_version, brainDir, completed: string[]}`. Atomic write via `.tmp` + `rename()` so a mid-write crash never leaves a partial JSON. `loadCheckpoint` returns `null` on: missing file, malformed JSON, brainDir mismatch (ran against a different brain), and the old positional format (logged to stderr before discard). `resumeFilter` returns `{toProcess, skippedCount}` — pure, no I/O, deterministic. `clearCheckpoint` is no-op-on-missing for clean-exit cleanup. Honors `GBRAIN_HOME` via `gbrainPath()` so `withEnv({GBRAIN_HOME: tmpdir})` test isolation works without monkey-patching fs. Best-effort persistence — `saveCheckpoint` logs warnings on write errors but never throws. - `src/core/sort-newest-first.ts` — single source of truth for the descending-lex sort that `gbrain import` and `gbrain sync` both apply. Mutates in place (Array.prototype.sort semantics), returns the same array reference for fluent chaining. Empty/single-element inputs short-circuit. Future ordering changes flip one line here instead of touching two CLI commands. Pinned by `test/sort-newest-first.test.ts` (descending order, mixed prefixes, empty, single-element, in-place-mutation contract). - `src/core/cycle.ts` — brain maintenance cycle primitive (9 phases). `runCycle(engine: BrainEngine | null, opts: CycleOpts): Promise` composes phases in semantic order: lint → backlinks → sync → synthesize → extract → patterns → recompute_emotional_weight → embed → orphans. `synthesize` runs after sync (cross-references see a fresh brain) and before extract (auto-link materializes its writes); `patterns` runs after extract so it reads a fresh graph (subagent put_page sets `ctx.remote=true` and skips auto-link/timeline by default, so extract is the canonical materialization); `recompute_emotional_weight` sees the union of `syncPagesAffected` + `synthesizeWrittenSlugs` incrementally, or all pages when neither anchor is set (full backfill via `gbrain dream --phase recompute_emotional_weight`). `CycleReport.schema_version: "1"` is stable; `totals` is additive (`pages_emotional_weight_recomputed`, `transcripts_processed`, `synth_pages_written`, `patterns_written`). Three callers: `gbrain dream` CLI, `gbrain autopilot` daemon inline path, the Minions `autopilot-cycle` handler. Coordination via `gbrain_cycle_locks` DB table + `~/.gbrain/cycle.lock` file lock with PID-liveness for PGLite. `yieldBetweenPhases` runs between phases; `yieldDuringPhase` is in-phase keepalive (synthesize/patterns renew the cycle-lock TTL during long waits). Engine nullable; lock-skip on read-only phase selections. `CycleOpts.signal?: AbortSignal` propagates the worker's abort signal with `checkAborted()` between every phase. `runPhaseSync` returns `pagesAffected` via `SyncPhaseResult` (threaded to `runPhaseExtract` as the 4th arg) and takes `willRunExtractPhase: boolean` setting `noExtract: phases.includes('extract')` so `gbrain dream --phase sync` doesn't silently lose extraction. `resolveSourceForDir(engine, brainDir)` threads `sourceId` to `performSync()` so sync reads the per-source `sources.last_commit` anchor (not the drift-prone global `config.sync.last_commit`). `CycleOpts.brainDir` is `string | null`; when null (checkout-less postgres/Supabase brain) the 6 filesystem phases (lint/backlinks/sync/synthesize/extract/patterns) skip with `details.reason: 'no_brain_dir'` and the DB-only phases run; `resolveSourceForDir` is null-tolerant. `cycleSourceId = opts.sourceId ?? resolveSourceForDir(engine, brainDir)` is the canonical per-source scope for `extract_facts`/`extract_atoms`/calibration — and for `synthesize` (#1586: threaded as `SynthesizePhaseOpts.sourceId` so synthesized pages land in the cycle's resolved source, not `'default'`) — so `gbrain dream --source repo-a` reconciles repo-a's facts even with no checkout (instead of scoping to `'default'` while stamping repo-a fresh). `deriveStatus` counts `edges_resolved`/`edges_ambiguous` as work so an edges-only cycle reports `ok` not `clean`; the `jobs.ts` `autopilot-cycle` + phase-wrapper handlers pass `null` (not `'.'`) when no repo is configured. The cycle is SPLIT for autopilot fan-out (#2194/#2227): `PHASE_SCOPE` partitions `ALL_PHASES` into `GLOBAL_PHASES` (brain-wide: embed/orphans/purge/resolve_symbol_edges/grade_takes/calibration_profile/synthesize_concepts/skillopt) and `NON_GLOBAL_PHASES` (source + mixed). Per-source `autopilot-cycle` jobs run only `NON_GLOBAL_PHASES` and stamp `last_source_cycle_at`; the single `autopilot-global-maintenance` job runs `GLOBAL_PHASES` (no `sourceId`) and stamps the brain-level `autopilot.last_global_at` config key (`LAST_GLOBAL_AT_KEY`). `last_full_cycle_at` is still written alongside `last_source_cycle_at` on a per-source success for doctor/legacy readers (no longer a gate for the brain-wide phases). Pinned by `test/dream-postgres.serial.test.ts` + `test/jobs-autopilot-cycle-braindir.serial.test.ts` + `test/autopilot-global-maintenance.test.ts`. diff --git a/src/cli.ts b/src/cli.ts index b12b776f0..5253c051d 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -2372,6 +2372,8 @@ IMPORT/EXPORT import [--no-embed] Import markdown directory sync [--repo ] [flags] Git-to-brain incremental sync sync --watch [--interval N] Continuous sync (loops until stopped) + sync --all --missing-path skip Classify sources whose local_path is absent + on this machine as skipped, not failed See also: autopilot --install (continuous daemon). export [--dir ./out/] Export to markdown export --restore-only [--repo

] Restore missing supabase-only files diff --git a/src/commands/sync.ts b/src/commands/sync.ts index 1c28b2280..86f230cbf 100644 --- a/src/commands/sync.ts +++ b/src/commands/sync.ts @@ -4169,12 +4169,22 @@ Options: connections per wave ≈ parallel × workers × 2 (per-file pool) + parent pool. Pass --parallel 1 to force serial. + --missing-path M (with --all) What to do when a source's local_path + does not exist on this machine: 'fail' (default — + loud, current behavior) or 'skip' (classify as + skipped_missing_path: ⊘ in the aggregate, excluded + from error_count and the rc=1 gate). Use skip on + brains whose sources were registered from more + than one machine. --json Emit a structured JSON envelope on stdout ({schema_version: 1, sources, parallel, - ok_count, error_count}). Human banners route to - stderr so '--json | jq' parses cleanly. - Exit codes: 0 = all sources ok, 1 = any error, - 2 = cost-prompt-not-confirmed. + ok_count, error_count, skipped_count}). Sources + skipped by --missing-path skip appear with + status 'skipped_missing_path' and their + local_path. Human banners route to stderr so + '--json | jq' parses cleanly. + Exit codes: 0 = all sources ok or skipped, + 1 = any error, 2 = cost-prompt-not-confirmed. --yes Accept any interactive prompts (CI / non-TTY). See also: @@ -4198,6 +4208,19 @@ See also: const noSchemaPack = args.includes('--no-schema-pack'); // v0.41.37.0 #1569 const includeGitignored = args.includes('--include-gitignored'); const syncAll = args.includes('--all'); + let missingPathMode: MissingPathMode = 'fail'; + try { + missingPathMode = parseMissingPathMode(args); + } catch (e) { + console.error(e instanceof Error ? e.message : String(e)); + process.exit(2); + } + if (missingPathMode !== 'fail' && !syncAll) { + // Single-source sync on a missing path should stay loud — an explicit + // `--source X` naming an absent checkout is an operator error, not a + // multi-machine artifact. Warn instead of silently ignoring the flag. + console.error('[gbrain] WARN: --missing-path only applies to `sync --all`; ignored here.'); + } const jsonOut = args.includes('--json'); const yesFlag = args.includes('--yes'); // v0.41.6.0 D3: lock-recovery flags. --break-lock (safe) verifies the @@ -4484,14 +4507,40 @@ See also: writeHuman(`Skipping ${disabledCount} disabled source(s).`); } - if (activeSources.length === 0) { + // --missing-path skip: classify sources whose checkout is not on this + // machine instead of failing them (see parseMissingPathMode's rationale). + // Under the default 'fail' this is a no-op and behavior is unchanged. + let skippedMissingPath: typeof activeSources = []; + let runnableSources = activeSources; + if (missingPathMode === 'skip') { + const parts = partitionMissingPathSources(activeSources, existsSync); + runnableSources = parts.runnable; + skippedMissingPath = parts.missing; + for (const src of skippedMissingPath) { + writeHuman(` ⊘ ${src.name}: skipped — local_path not present on this host (${src.local_path})`); + } + if (skippedMissingPath.length > 0) { + writeHuman(`Skipped ${skippedMissingPath.length} source(s) whose local_path is not present on this host (--missing-path skip).`); + } + } + + if (runnableSources.length === 0) { if (jsonOut) { console.log(JSON.stringify({ schema_version: 1, - sources: [], + sources: skippedMissingPath + .slice() + .sort((a, b) => a.id.localeCompare(b.id)) + .map((s) => ({ + source_id: s.id, + name: s.name, + status: 'skipped_missing_path', + local_path: s.local_path, + })), parallel: 0, ok_count: 0, error_count: 0, + skipped_count: skippedMissingPath.length, })); } return; @@ -4501,11 +4550,20 @@ See also: type PerSourceResult = { sourceId: string; sourceName: string; - status: 'ok' | 'error'; + status: 'ok' | 'error' | 'skipped_missing_path'; result?: SyncResult; error?: string; + localPath?: string; }; const perSourceResults: PerSourceResult[] = []; + for (const src of skippedMissingPath) { + perSourceResults.push({ + sourceId: src.id, + sourceName: src.name, + status: 'skipped_missing_path', + localPath: src.local_path ?? undefined, + }); + } // #1633 (Part B): one shared SIGINT controller for the whole --all fan-out. // process-cleanup.ts doesn't own SIGINT, so without this Ctrl-C hard-cuts the @@ -4615,7 +4673,7 @@ See also: }; const parallelEligible = - v2Enabled && !serialFlag && engine.kind !== 'pglite' && activeSources.length > 1; + v2Enabled && !serialFlag && engine.kind !== 'pglite' && runnableSources.length > 1; // v0.42.42.0 (#2139, D13C): the v0.40.6.0 (D15) refusal of --skip-failed / // --retry-failed under parallel sync is LIFTED. It existed because the @@ -4629,7 +4687,7 @@ See also: // know how the run was actually dispatched. 1 in the serial fallback, // capped at min(sourceCount, --max-sources, 8) in the parallel path. const effectiveParallel = parallelEligible - ? Math.min(activeSources.length, maxSources ?? 8) + ? Math.min(runnableSources.length, maxSources ?? 8) : 1; process.on('SIGINT', onAllSigint); @@ -4653,8 +4711,8 @@ See also: ); } - writeHuman(`\nParallel sync: ${activeSources.length} sources, ${cap} concurrent workers.\n`); - const results = await pMapAllSettled(activeSources, cap, async (src) => { + writeHuman(`\nParallel sync: ${runnableSources.length} sources, ${cap} concurrent workers.\n`); + const results = await pMapAllSettled(runnableSources, cap, async (src) => { const r = await runOne(src); return { name: src.name, result: r }; }); @@ -4662,7 +4720,7 @@ See also: writeHuman('\n--- sync --all aggregate ---'); for (let i = 0; i < results.length; i++) { const r = results[i]; - const src = activeSources[i]; + const src = runnableSources[i]; if (r.status === 'fulfilled') { writeHuman(` ✓ ${src.name}: ${r.value.result.status} (added=${r.value.result.added}, modified=${r.value.result.modified}, deleted=${r.value.result.deleted})`); perSourceResults.push({ @@ -4683,7 +4741,7 @@ See also: } } } else { - for (const src of activeSources) { + for (const src of runnableSources) { writeHuman(`\n--- Syncing source: ${src.name} ---`); try { const result = await runOne(src); @@ -4723,6 +4781,7 @@ See also: source_id: r.sourceId, name: r.sourceName, status: r.status, + ...(r.localPath ? { local_path: r.localPath } : {}), ...(r.result ? { sync_status: r.result.status, // #3068: surface the partial reason (e.g. pull_failed) so JSON @@ -4742,6 +4801,7 @@ See also: parallel: effectiveParallel, ok_count: okCount, error_count: errCount, + skipped_count: perSourceResults.filter((r) => r.status === 'skipped_missing_path').length, })); } @@ -4938,6 +4998,63 @@ See also: } } +/** Mode for `sync --all --missing-path`: what to do when a source's + * local_path does not exist on this machine. */ +export type MissingPathMode = 'fail' | 'skip'; + +/** + * Parse `--missing-path ` (default: fail). + * + * Why the flag exists: `sources.local_path` is machine-specific state in a + * brain-wide table. Any brain whose sources were registered from more than + * one machine — or a sanctioned setup mid-migration (topologies.md Topology 2, + * or the system-of-record git flow before every repo is cloned here) — has + * sources whose checkout simply is not present on the machine running + * `sync --all`. Each used to surface as a hard failure ("Not a git + * repository: ") and force rc=1 on every run; on one observed fleet + * that was 12 phantom failures per hour, which trains operators to ignore + * the exit code. + * + * The DEFAULT stays `fail`: on a single-machine brain a missing local_path + * usually means an unmounted volume or a deleted checkout, and silently + * skipping it would hide real data loss. Skip is an explicit opt-in. + * + * Throws on a bad/absent value with a paste-ready hint (caller converts to + * stderr + exit 2, same as other flag-misuse exits). + */ +export function parseMissingPathMode(args: string[]): MissingPathMode { + const idx = args.indexOf('--missing-path'); + if (idx === -1) return 'fail'; + const val = args[idx + 1]; + if (val === 'fail' || val === 'skip') return val; + throw new Error( + `--missing-path expects 'fail' or 'skip', got: ${val ?? '(nothing)'}. ` + + `Use \`--missing-path skip\` to classify sources whose local_path is not ` + + `present on this machine as skipped instead of failed, or \`--missing-path ` + + `fail\` (the default) to keep them loud.`, + ); +} + +/** + * Partition `--all` sources by whether their local_path exists on THIS + * machine. Classification is driven only by the injected predicate so tests + * never touch the filesystem. A null local_path passes through as runnable — + * pure-DB sources are already excluded from `--all` by the + * `local_path IS NOT NULL` SELECT; this is defensive, not load-bearing. + */ +export function partitionMissingPathSources( + sources: T[], + pathExists: (p: string) => boolean, +): { runnable: T[]; missing: T[] } { + const runnable: T[] = []; + const missing: T[] = []; + for (const s of sources) { + if (s.local_path != null && !pathExists(s.local_path)) missing.push(s); + else runnable.push(s); + } + return { runnable, missing }; +} + /** * v0.40.3.0 — resolve effective per-source concurrency for `sync --all`. * diff --git a/test/sync-all-missing-path.test.ts b/test/sync-all-missing-path.test.ts new file mode 100644 index 000000000..f38d65651 --- /dev/null +++ b/test/sync-all-missing-path.test.ts @@ -0,0 +1,111 @@ +/** + * Tests for `sync --all --missing-path `. + * + * Why this exists: + * `sources.local_path` is machine-specific state in a brain-wide table. + * Any brain whose sources were registered from more than one machine — + * or a sanctioned setup mid-migration (docs/architecture/topologies.md + * Topology 2, or the system-of-record git flow before every repo is + * cloned) — has sources whose checkout is simply not present on the + * machine running `sync --all`. Today each of those is reported as a + * hard per-source failure ("Not a git repository: ") and the run + * exits rc=1, every run. On one observed fleet that was 12 phantom + * failures per hour with zero actionable signal, which trains operators + * to ignore the exit code — the worst possible property for a cron. + * + * `--missing-path skip` classifies those sources honestly instead: + * status `skipped_missing_path` in the --json envelope, a ⊘ line in the + * human aggregate, excluded from error_count and from the rc=1 gate. + * + * The DEFAULT stays `fail`: on a single-machine brain a missing + * local_path usually means an unmounted volume or a deleted checkout, + * and silently skipping it would hide real data loss. Skip is opt-in. + * + * These tests pin the two pure helpers (same style as + * sync-all-parallel.test.ts — no DB, no fs): + * 1. parseMissingPathMode(): flag parsing, default, loud rejection of + * bad values (paste-ready hint names the flag and both values). + * 2. partitionMissingPathSources(): classification is driven ONLY by + * the injected pathExists predicate; null local_path passes through + * as runnable (pure-DB sources are already excluded from --all by + * the local_path IS NOT NULL SELECT — defensive, not load-bearing). + */ +import { describe, expect, test } from 'bun:test'; +import { + parseMissingPathMode, + partitionMissingPathSources, +} from '../src/commands/sync.ts'; + +// ── parseMissingPathMode ──────────────────────────────────────────── + +describe('parseMissingPathMode', () => { + test('defaults to fail when the flag is absent', () => { + expect(parseMissingPathMode([])).toBe('fail'); + expect(parseMissingPathMode(['--all', '--no-embed'])).toBe('fail'); + }); + + test('parses skip and fail explicitly', () => { + expect(parseMissingPathMode(['--all', '--missing-path', 'skip'])).toBe('skip'); + expect(parseMissingPathMode(['--all', '--missing-path', 'fail'])).toBe('fail'); + }); + + test('rejects an unknown value with a paste-ready hint', () => { + expect(() => parseMissingPathMode(['--missing-path', 'ignore'])) + .toThrow(/--missing-path.*(fail|skip)/); + }); + + test('rejects a dangling flag (no value)', () => { + expect(() => parseMissingPathMode(['--all', '--missing-path'])) + .toThrow(/--missing-path/); + }); + + test('does not swallow a following flag as its value', () => { + // `--missing-path --json` is a mistake, not "mode --json". + expect(() => parseMissingPathMode(['--missing-path', '--json'])) + .toThrow(/--missing-path/); + }); +}); + +// ── partitionMissingPathSources ───────────────────────────────────── + +type Src = { id: string; local_path: string | null }; +const src = (id: string, local_path: string | null): Src => ({ id, local_path }); + +describe('partitionMissingPathSources', () => { + test('splits sources by the injected pathExists predicate', () => { + const sources = [src('here', '/present'), src('elsewhere', '/absent')]; + const { runnable, missing } = partitionMissingPathSources( + sources, (p) => p === '/present'); + expect(runnable.map((s) => s.id)).toEqual(['here']); + expect(missing.map((s) => s.id)).toEqual(['elsewhere']); + }); + + test('null local_path stays runnable (pure-DB sources are not "missing")', () => { + const { runnable, missing } = partitionMissingPathSources( + [src('db-only', null)], () => false); + expect(runnable.map((s) => s.id)).toEqual(['db-only']); + expect(missing).toEqual([]); + }); + + test('all-missing and empty inputs are well-formed', () => { + const allMissing = partitionMissingPathSources( + [src('a', '/x'), src('b', '/y')], () => false); + expect(allMissing.runnable).toEqual([]); + expect(allMissing.missing.map((s) => s.id)).toEqual(['a', 'b']); + + const empty = partitionMissingPathSources([], () => true); + expect(empty.runnable).toEqual([]); + expect(empty.missing).toEqual([]); + }); + + test('never consults the predicate for null paths and preserves order', () => { + const asked: string[] = []; + const sources = [src('a', '/1'), src('b', null), src('c', '/2')]; + const { runnable } = partitionMissingPathSources(sources, (p) => { + asked.push(p); + return true; + }); + expect(asked).toEqual(['/1', '/2']); + expect(runnable.map((s) => s.id)).toEqual(['a', 'b', 'c']); + }); +});