From 6f26d5e4df45e1973c1a6df8361735f70d45e6ce Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 30 May 2026 14:56:26 -0700 Subject: [PATCH] =?UTF-8?q?v0.41.37.0=20fix:=20critical=20fix=20wave=20?= =?UTF-8?q?=E2=80=94=20reindex=20tag=20wipe,=20grandfather=20hang,=20Windo?= =?UTF-8?q?ws=20migration=20spawn,=20sync=20ReDoS=20(#1621=20#1581=20#1605?= =?UTF-8?q?=20#1569)=20(#1665)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(reindex): add-only tag reconciliation + DB-only re-chunk preserves frontmatter (#1621) reindex --markdown and re-import no longer wipe DB-side enrichment tags. Tag reconciliation is now ADD-ONLY (import-file.ts): re-import adds current frontmatter tags and never deletes, so auto/dream/signal-detector tags survive. The reindex DB-only fallback reconstructs full markdown via serializeMarkdown so re-chunking a page with no on-disk source preserves frontmatter/title/timeline. * fix(migrations): v0.13.1 grandfather chunked, source-safe, soft-delete-filtered (#1581) phaseCGrandfather rewritten from a per-page getPage+putPage loop (which hung 70+ min on an 82K-page PGLite brain) to a chunked bulk SQL pass keyed on pages.id (NOT slug — slug isn't globally unique), filtering deleted_at IS NULL, with a batched rollback log carrying source identity. * fix(migrations): run schema phases in-process to fix Windows getaddrinfo ENOTFOUND (#1605) The 9 'gbrain init --migrate-only' execSync spawns died on Windows+bun+Supabase (child DNS resolution). runMigrateOnlyCore (extracted from initMigrateOnly) runs the schema bring-up in-process for all engines, unblocking schema_version advancement. Includes async-call-site audit, a wall-clock guard, and a runGbrainSubprocess stderr-capture wrapper for the remaining backfill spawns. * fix(sync): ReDoS hardening + diagnostics for schema-pack regexes (#1569) Input-length cap in runRegexBounded + route the unbounded link-inference path through it (closes the only no-timeout ReDoS hole); star-height lint rule warns on nested-quantifier patterns; --no-schema-pack sync escape hatch; GBRAIN_SYNC_TRACE per-file begin heartbeat; PGLite serve/sync concurrency doc. Defensive hardening + diagnostics — the deterministic ~3100-file wedge root cause remains open (no repro). * docs(todos): file v0.41.37.0 fix-wave follow-ups (#1621/#1605/#1569) * chore: bump version and changelog (v0.41.37.0) Co-Authored-By: Claude Opus 4.8 (1M context) * docs: sync README + CLAUDE.md for v0.41.37.0 critical fix wave Add reindex add-only tag reconciliation (#1621), v0.13.1 grandfather + Windows in-process migration (#1581/#1605), and schema-pack ReDoS hardening + sync --no-schema-pack / GBRAIN_SYNC_TRACE triage (#1569) to CLAUDE.md key-files annotations and README Troubleshooting. Regenerated llms-full.txt. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(ci): bump llms-full.txt budget 700KB→750KB (CLAUDE.md crossed 700KB after master merge) The build-llms size-budget test failed: llms-full.txt is 703,244 bytes after the v0.41.37.0 key-files annotations merged on top of master's v0.41.34/35/36 CLAUDE.md additions. Matches the v0.41.9.0 precedent (600→700); the single-fetch bundle still fits comfortably in modern long-context models. --------- Co-authored-by: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 117 +++++++++++++ CLAUDE.md | 11 +- README.md | 47 +++++ TODOS.md | 12 ++ VERSION | 2 +- docs/architecture/serve-sync-concurrency.md | 54 ++++++ llms-full.txt | 58 ++++++- package.json | 2 +- scripts/llms-config.ts | 11 +- src/commands/init.ts | 47 ++--- src/commands/migrations/in-process.ts | 139 +++++++++++++++ src/commands/migrations/v0_11_0.ts | 31 +--- src/commands/migrations/v0_12_0.ts | 12 +- src/commands/migrations/v0_12_2.ts | 10 +- src/commands/migrations/v0_13_0.ts | 14 +- src/commands/migrations/v0_13_1.ts | 183 +++++++++++--------- src/commands/migrations/v0_16_0.ts | 8 +- src/commands/migrations/v0_18_0.ts | 8 +- src/commands/migrations/v0_18_1.ts | 8 +- src/commands/migrations/v0_21_0.ts | 13 +- src/commands/migrations/v0_29_1.ts | 13 +- src/commands/reindex.ts | 21 ++- src/commands/sync.ts | 37 +++- src/core/import-file.ts | 25 ++- src/core/schema-pack/link-inference.ts | 14 +- src/core/schema-pack/lint-rules.ts | 29 ++++ src/core/schema-pack/redos-guard.ts | 29 ++++ test/fix-wave-structural.test.ts | 13 +- test/import-file.test.ts | 12 +- test/migration-in-process.serial.test.ts | 112 ++++++++++++ test/migrations-v0_13_0.test.ts | 14 +- test/migrations-v0_13_1-grandfather.test.ts | 144 +++++++++++++++ test/migrations-v0_16_0.test.ts | 5 +- test/redos-hardening.test.ts | 96 ++++++++++ test/reindex-preserve-tags.test.ts | 75 ++++++++ test/schema-pack-lint-rules.test.ts | 7 +- 36 files changed, 1201 insertions(+), 232 deletions(-) create mode 100644 docs/architecture/serve-sync-concurrency.md create mode 100644 src/commands/migrations/in-process.ts create mode 100644 test/migration-in-process.serial.test.ts create mode 100644 test/migrations-v0_13_1-grandfather.test.ts create mode 100644 test/redos-hardening.test.ts create mode 100644 test/reindex-preserve-tags.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 8249ae377..332670638 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,110 @@ All notable changes to GBrain will be documented in this file. +## [0.41.37.0] - 2026-05-30 + +**A four-bug critical fix wave: your tags stop disappearing on reindex, a +migration that hung forever on big brains now finishes in seconds, Windows +users stuck mid-upgrade get unstuck, and a class of sync-time regex blowups is +capped.** + +If you run `gbrain reindex --markdown` (or a cron that does), it was quietly +wiping tags. Specifically: any tag that wasn't written in the page's `.md` +frontmatter — the ones your dream cycle, signal-detector, or auto-tagging added +straight to the database — got deleted on every reindex. One reporter watched +their distinct-tag count fall from ~848 to ~100. That's fixed: reindex and +re-import are now **add-only**. They add whatever tags are in the frontmatter +and never delete, so database-side tags survive. (The trade-off: removing a tag +from frontmatter no longer removes it from the DB on the next sync — additive +metadata, low harm, and a provenance-column fix is filed for later.) + +If you're on a large brain and `gbrain apply-migrations` ever hung, the v0.13.1 +"grandfather" step was the culprit. On an 82K-page brain it pinned a CPU core +for over an hour after ~4,200 pages and never finished. It walked every page one +at a time. It now does the same work as a handful of bulk SQL statements and +finishes in a second or two. It's also now safe on multi-source brains (it keys +on the page's unique id, not its slug) and leaves soft-deleted pages alone. + +If you're on Windows with a Supabase brain and got stuck at an old +`schema_version` while the binary kept upgrading, this one's for you. The +migration step shelled out to a child `gbrain init --migrate-only` process, and +on Windows+bun+Supabase that child died with `getaddrinfo ENOTFOUND` before it +could connect — even though the parent connected fine. The schema bring-up now +runs in-process (no child process), so the upgrade actually advances. The +remaining data-backfill steps that still shell out now surface the real error +instead of a bare "Command failed," so the next Windows issue is diagnosable. + +And if `gbrain sync` ever pinned CPU on a brain with a custom schema pack, we +capped the blast radius: a pack's link-inference regex now has a hard +input-length limit (catastrophic backtracking needs a long input; we don't give +it one), and `gbrain schema lint` warns when a pack regex has the classic +exponential shape like `(a+)+`. New escape hatch: `gbrain sync --no-schema-pack` +completes a sync without running any pack regex. New diagnostic: +`GBRAIN_SYNC_TRACE=1 gbrain sync` prints the file being imported so a hang names +the culprit. (Honest scope: this is hardening plus diagnostics. The specific +deterministic wedge one reporter hit at ~3100 files on a 56K-file brain is not +root-caused yet — their repro sample is the next step.) + +### What you might need to do + +- **Lost tags from a past reindex?** They aren't recoverable from frontmatter + (they were never there). Re-run whatever enrichment originally created them. + Going forward, reindex won't wipe them. +- **Stuck mid-migration (Windows or a hung large-brain upgrade)?** + `gbrain upgrade` then `gbrain apply-migrations --yes` should now complete. + Verify with `gbrain doctor`. +- **Sync wedging on a schema-pack brain?** `gbrain sync --no-schema-pack` to get + unblocked, then `gbrain schema lint` to find the bad regex. + +### Itemized changes + +- **#1621 — reindex/sync no longer wipe DB-side tags.** Tag reconciliation in + `src/core/import-file.ts` is now add-only (adds current frontmatter tags via + the idempotent `ON CONFLICT DO NOTHING` `addTag`, never deletes). The + `gbrain reindex --markdown` DB-only fallback (pages with no on-disk source) + reconstructs full markdown via `serializeMarkdown(page)` before re-importing, + so re-chunking preserves frontmatter, title, and timeline instead of + overwriting them from body-only content. +- **#1581 — v0.13.1 grandfather migration no longer hangs.** `phaseCGrandfather` + rewritten from a per-page `getPage`+`putPage` loop to a chunked bulk SQL pass + keyed on `pages.id` (slug is not globally unique — a slug-keyed UPDATE would + cross-contaminate same-slug pages in other sources), filtering + `deleted_at IS NULL`, with a batched rollback log carrying `{id, slug, + source_id, pre_frontmatter}`. +- **#1605 — Windows migration upgrade unblocked.** New + `src/commands/migrations/in-process.ts` exports `runMigrateOnlyCore` (extracted + from `init.ts`'s `initMigrateOnly`, single source of truth so the + configure-gateway-before-initSchema fix can't drift). The 9 + `gbrain init --migrate-only` schema-phase spawns across the migration + orchestrators now run in-process for every engine, with a wall-clock guard. + Remaining `extract`/`repair` spawns route through `runGbrainSubprocess`, which + captures child stderr (large `maxBuffer`) and folds it into the failure detail. +- **#1569 — schema-pack ReDoS hardening + sync diagnostics.** Input-length cap + (`MAX_REGEX_INPUT_CHARS`, default 64K, env-overridable) in + `redos-guard.ts:runRegexBounded`; the previously-unbounded + `link-inference.ts` path now routes through it. New `link_regex_catastrophic_backtrack` + lint rule (warning) flags nested-quantifier patterns. New `gbrain sync + --no-schema-pack` flag (threaded through the `--all` per-source path). New + `GBRAIN_SYNC_TRACE=1` per-file begin heartbeat. New + `docs/architecture/serve-sync-concurrency.md`. + +### For contributors + +- 8 new/updated test files (28 new cases): `test/reindex-preserve-tags.test.ts`, + `test/migrations-v0_13_1-grandfather.test.ts` (multi-source + soft-delete + + large-N), `test/migration-in-process.serial.test.ts`, + `test/redos-hardening.test.ts`, plus updates to `import-file`, + `migrations-v0_13_0`, `migrations-v0_16_0`, `schema-pack-lint-rules`, and + `fix-wave-structural` tests for the changed contracts. +- `phaseASchema` in the 9 migration orchestrators is now `async` and awaited at + every call site (a structural source-grep guard in + `migration-in-process.serial.test.ts` bans reintroducing the + `execSync('gbrain init --migrate-only')` spawn). + +## To take advantage of v0.41.37.0 + +`gbrain upgrade` should do this automatically. If it didn't, or if `gbrain +doctor` warns about a partial migration: ## [0.41.36.0] - 2026-05-30 **Your MCP clients can now see and use your agent's skills. Point Codex desktop, @@ -224,6 +328,19 @@ columns) automatically. If `gbrain doctor` warns about a partial migration: ```bash gbrain apply-migrations --yes ``` +2. **No agent action is required** — these are bug fixes; the mechanical schema + side is handled by the orchestrator. +3. **Verify the outcome:** + ```bash + gbrain doctor + gbrain stats + ``` + On a brain that previously hung on v0.13.1, the cascade should now complete in + seconds. On Windows + Supabase, `schema_version` should reach head. +4. **If any step fails or numbers look wrong,** file an issue at + https://github.com/garrytan/gbrain/issues with the output of `gbrain doctor` + and the contents of `~/.gbrain/upgrade-errors.jsonl` if it exists. + 2. **Backfill aliases for pages you already have** (one time): ```bash gbrain reindex --aliases diff --git a/CLAUDE.md b/CLAUDE.md index 82e31e249..8c568a3c9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -58,13 +58,13 @@ strict behavior when unset. - `src/core/audit-slug-fallback.ts` (v0.32.7 CJK wave) — Weekly ISO-week-rotated audit JSONL at `~/.gbrain/audit/slug-fallback-YYYY-Www.jsonl`. `logSlugFallback(slug, sourcePath)` fires when `importFromFile` falls back to a frontmatter slug because `slugifyPath` returned empty (emoji / Thai / Arabic / non-CJK exotic-script filenames). `readRecentSlugFallbacks(days)` reads the last N days for `gbrain doctor`'s `slug_fallback_audit` check. Honors `GBRAIN_AUDIT_DIR` via the shared `resolveAuditDir()` from shell-audit.ts. Separate surface from `sync-failures.jsonl` per codex outside-voice review — that file carries bookmark-gating semantics that info events shouldn't trigger. - `src/core/embedding-pricing.ts` (v0.32.7 CJK wave) — `EMBEDDING_PRICING` map keyed `provider:model` for the post-upgrade reindex cost estimate. Sibling to `anthropic-pricing.ts`. Entries: OpenAI text-embedding-3-large ($0.13/1M), 3-small ($0.02/1M), ada-002 ($0.10/1M), Voyage 3-large ($0.18/1M), 3 ($0.06/1M). `lookupEmbeddingPrice(modelString)` returns a tagged union (`known` with price + `unknown` with provider name); `estimateCostFromChars(charCount, pricePerMTok)` uses 3.5 chars/token approximation. Unknown providers degrade gracefully to "estimate unavailable" instead of fabricating numbers. - `src/core/post-upgrade-reembed.ts` (v0.32.7 CJK wave) — Pure functions backing the `gbrain upgrade` chunker-bump cost prompt. `computeReembedEstimate(engine, model)` queries real SQL (`COUNT(*)` + `COALESCE(SUM(LENGTH(compiled_truth)) + SUM(LENGTH(timeline)), 0)`) on `pages WHERE chunker_version < MARKDOWN_CHUNKER_VERSION`. `formatReembedPrompt(est, graceSeconds)` is the stderr-line formatter. `runPostUpgradeReembedPrompt(engine, model, opts)` orchestrates the 10-second Ctrl-C window; TTY-only wait (non-TTY auto-proceeds for CI / cron); `GBRAIN_NO_REEMBED=1` bails out with a doctor-warning marker; `GBRAIN_REEMBED_GRACE_SECONDS=0` skips the wait. -- `src/commands/reindex.ts` (v0.32.7 CJK wave) — `gbrain reindex --markdown [--limit N] [--dry-run] [--json] [--no-embed] [--repo PATH]`. Walks `pages WHERE page_kind = 'markdown' AND chunker_version < MARKDOWN_CHUNKER_VERSION` in 100-row batches, ordered by id. Rows with non-null `source_path` re-import via `importFromFile`; rows without fall back to `importFromContent` against the stored `compiled_truth`. **Both paths pass `forceRechunk: true`** to bypass `importFromContent`'s `content_hash` short-circuit — without that flag (codex post-merge F1), the chunker version bump never reaches pages whose source content hasn't changed since last sync, AND master's v0.32.2 stripFactsFence privacy strip never applies to pre-strip chunks. Idempotent — partial-completion re-runs pick up where they left off via id-ordered batches. Wired into `src/commands/upgrade.ts:runPostUpgrade` after `apply-migrations`. +- `src/commands/reindex.ts` (v0.32.7 CJK wave) — `gbrain reindex --markdown [--limit N] [--dry-run] [--json] [--no-embed] [--repo PATH]`. Walks `pages WHERE page_kind = 'markdown' AND chunker_version < MARKDOWN_CHUNKER_VERSION` in 100-row batches, ordered by id. Rows with non-null `source_path` re-import via `importFromFile`; rows without fall back to `importFromContent` against the stored `compiled_truth`. **Both paths pass `forceRechunk: true`** to bypass `importFromContent`'s `content_hash` short-circuit — without that flag (codex post-merge F1), the chunker version bump never reaches pages whose source content hasn't changed since last sync, AND master's v0.32.2 stripFactsFence privacy strip never applies to pre-strip chunks. Idempotent — partial-completion re-runs pick up where they left off via id-ordered batches. Wired into `src/commands/upgrade.ts:runPostUpgrade` after `apply-migrations`. **v0.41.37.0 (#1621):** the DB-only fallback (no source file on disk) no longer passes body-only `compiled_truth` to `importFromContent` — that path re-parsed with EMPTY frontmatter and OVERWROTE the page's real frontmatter / title / timeline (codex catch). It now `getPage` + `getTags`, reconstructs the FULL markdown via `serializeMarkdown(frontmatter, compiled_truth, timeline, {type, title, tags})`, and re-imports THAT, so re-chunking a DB-only page preserves everything while still bumping `chunker_version`. Pinned by `test/reindex-preserve-tags.test.ts`. - `src/commands/reindex-code.ts` (v0.21.0 Cathedral II E2, extended v0.37.3.0) — `gbrain reindex --code [--source ID] [--dry-run] [--yes] [--json] [--force] [--no-embed]`. Walks `pages WHERE type = 'code'` in 100-row batches, replays through `importCodeFile` for chunk + embed + content_hash folding. Idempotent unless `--force` bypasses the content_hash early-return. **v0.37.3.0:** cost-preview model field now reads `getEmbeddingModelName()` from the gateway instead of the back-compat `EMBEDDING_MODEL` constant — preview reflects what the gateway will actually embed with. Same wave adds an informational stderr nudge inside `runReindexCode` (not the CLI wrapper, so dry-run + execute both surface it): when the configured embedding model isn't code-tuned (allowlist currently `{'voyage-code-3'}`, case-insensitive bare match against gateway-returned name), prints a 4-line recommendation to switch to `voyage:voyage-code-3`. Suppress with `GBRAIN_NO_CODE_MODEL_NUDGE=1`, `--no-embed`, or `--json`. Pure `shouldNudgeCodeModel(bareName)` helper returns a tagged `NudgeDecision` union; the helper takes the bare model name (matches gateway return shape) and emits qualified `voyage:voyage-code-3` for the paste-ready `gbrain config set` line. Pinned by `test/ai/voyage-code-3-recipe.test.ts`, `test/reindex-code-nudge.serial.test.ts`, and `test/reindex-code-model-source.serial.test.ts` (the latter is the IRON-RULE regression for the cost-preview fix). - `src/commands/sync.ts:resolveSlugByPathOrSourcePath` (v0.32.7 CJK wave, codex post-merge F4) — Resolves a slug by `pages.source_path` first (returns the stored slug for frontmatter-fallback pages whose path doesn't derive a slug), then falls back to `resolveSlugForPath(path)`. Threaded into all 4 delete/rename call sites (`performSync`'s un-syncable cleanup at ~:531, deletes at ~:603, rename oldSlug at ~:622). Without this, emoji-only / Thai / Arabic filenames whose slug came from frontmatter would orphan on delete/rename (the delete path would compute the wrong path-derived slug). Best-effort query — pre-migration brains fall through to the legacy path. - `src/core/utils.ts` — Shared SQL utilities extracted from postgres-engine.ts. Exports `parseEmbedding(value)` (throws on unknown input, used by migration + ingest paths where data integrity matters) and as of v0.12.3 `tryParseEmbedding(value)` (returns `null` + warns once per process, used by search/rescore paths where availability matters more than strictness). **v0.26.9 (D14):** adds `isUndefinedColumnError(err)` predicate — pattern-matches Postgres SQLSTATE 42703 / "column ... does not exist" with engine-driver shape variation tolerated. Replaces bare `catch {}` blocks in `oauth-provider.ts` so genuine errors (lock timeout, network blip, permission denied) propagate while column-missing falls through to the legacy fallback path. Reusable from any future code that needs the same column-existence probe semantics. **v0.32.8 (PR #860):** adds `validateSourceId(id)` that throws on anything outside `^[a-z0-9_-]+$`. Used by the per-source disk-layout fix in patterns.ts/synthesize.ts before any `join(brainDir, '.sources', source_id, slug+'.md')` call so source_id can't traverse out of brainDir. `rowToPage` updated to populate the now-required `Page.source_id` field from the SELECT projection (`scripts/check-source-id-projection.sh` enforces that every projection feeding `rowToPage` includes the column). - `src/core/db.ts` — Connection management, schema initialization. v0.22.1 (#363, contributed by @orendi84): `resolveSessionTimeouts()` returns `statement_timeout` + `idle_in_transaction_session_timeout` (defaults: 5min each, env-overridable via `GBRAIN_STATEMENT_TIMEOUT` / `GBRAIN_IDLE_TX_TIMEOUT` / `GBRAIN_CLIENT_CHECK_INTERVAL`). Both `connect()` (module singleton) and `PostgresEngine.connect()` (worker pool) consume the result via postgres.js's `connection` option, sending GUCs as startup parameters that survive PgBouncer transaction mode (unlike the prior `setSessionDefaults` post-pool SET, kept as a back-compat no-op shim). - `src/commands/migrate-engine.ts` — Bidirectional engine migration (`gbrain migrate --to supabase/pglite`) -- `src/core/import-file.ts` — importFromFile + importFromContent (chunk + embed + tags). **v0.41.31.0:** `importFromContent` and `importCodeFile` stamp `pages.embedding_signature` via `setPageEmbeddingSignature(slug, {sourceId, signature: currentEmbeddingSignature()})` when the import actually embedded (not `--no-embed`) — covers the inline import/sync path so a model/dims swap is detectable as stale. `importCodeFile` only stamps when every chunk was freshly embedded this call (`needsEmbedIndexes.length === chunks.length`); mixed reuse-by-hash pages stay unstamped (`reindex --code --force` / `embed --stale` handle the swap for those). +- `src/core/import-file.ts` — importFromFile + importFromContent (chunk + embed + tags). **v0.41.31.0:** `importFromContent` and `importCodeFile` stamp `pages.embedding_signature` via `setPageEmbeddingSignature(slug, {sourceId, signature: currentEmbeddingSignature()})` when the import actually embedded (not `--no-embed`) — covers the inline import/sync path so a model/dims swap is detectable as stale. `importCodeFile` only stamps when every chunk was freshly embedded this call (`needsEmbedIndexes.length === chunks.length`); mixed reuse-by-hash pages stay unstamped (`reindex --code --force` / `embed --stale` handle the swap for those). **v0.41.37.0 (#1621):** `importFromContent`'s tag reconciliation is now ADD-ONLY. The pre-fix "remove every existing tag not in current frontmatter, then add current" logic wiped ALL DB-side enrichment tags (auto-tag / dream synthesize / signal-detector) on every re-import — most visibly under `gbrain reindex --markdown`, which re-imports every page with `forceRechunk`. The `tags` table has no provenance column and frontmatter tags are stripped from stored `pages.frontmatter` (markdown.ts:118), so at re-import time a frontmatter-origin tag can't be distinguished from a DB-enrichment tag — deletion is unsafe. Now it only `addTag` (idempotent, ON CONFLICT DO NOTHING). Accepted trade-off: removing a tag from frontmatter no longer removes it from the DB on next sync (needs a `tag_source` provenance column, deferred — TODOS.md #1621-followup). Pinned by `test/reindex-preserve-tags.test.ts` + `test/import-file.test.ts`. - `src/core/sync.ts` — Pure sync functions (manifest parsing, filtering, slug conversion). **v0.35.5.0:** new exported `pruneDir(name: string): boolean` helper is the single source of truth for descent-time directory exclusion across walkers. Blocks `node_modules` (no leading dot, so pre-v0.35.5 walkers slipped through and inflated MISSING_OPEN counts via vendor packages), dot-prefix dirs, `ops/`, and `*.raw` sidecars. `isSyncable` now applies it per path segment; `walkMarkdownFiles` in `src/commands/extract.ts` and `listTextFiles` in `src/core/cycle/transcript-discovery.ts` consult it BEFORE recursing so the IO cost of walking thousands of vendor files is saved. Closes #923 + #202. `manageGitignore` worktree fix in same wave: discriminator now matches the gitdir path segment (`/modules/` = submodule, `/worktrees/` = worktree, per Git's documented layout) instead of the legacy absolute-vs-relative check that misclassified absorbed submodules and worktrees both. Conductor worktrees are first-class repos and now get `.gitignore` management for storage-tiering. Closes #889. v0.22.12 (#500, foundation by @wintermute via #501): `classifyErrorCode(errorMsg)` regex-based classifier with 12 codes (`SLUG_MISMATCH`, `YAML_PARSE`, `YAML_DUPLICATE_KEY`, `MISSING_OPEN`, `MISSING_CLOSE`, `NESTED_QUOTES`, `EMPTY_FRONTMATTER`, `NULL_BYTES`, `INVALID_UTF8`, `STATEMENT_TIMEOUT`, `FILE_TOO_LARGE`, `SYMLINK_NOT_ALLOWED`) plus `UNKNOWN` fallback. `summarizeFailuresByCode(failures)` returns sorted `[{code, count}]`. `code?` optional field on `SyncFailure`; backfilled at ack time on pre-v0.22.12 entries. `acknowledgeSyncFailures()` returns `AcknowledgeResult { count, summary }`. Three regexes (`MISSING_OPEN`, `MISSING_CLOSE`, `EMPTY_FRONTMATTER`) broadened to match actual `markdown.ts:159-244` validator message strings, not just the literal code-name prefix. `FILE_TOO_LARGE` covers all three production size sites in `import-file.ts:199, 352, 401`; `SYMLINK_NOT_ALLOWED` covers the rejection at `:347`. Closes the silent-skip pattern that motivated #500. - `src/core/storage.ts` — Pluggable storage interface (S3, Supabase Storage, local) - `src/core/storage-config.ts` (v0.22.11) — Storage tiering: `loadStorageConfig` reads `gbrain.yml`, normalizes deprecated keys (`git_tracked` / `supabase_only`) to canonical (`db_tracked` / `db_only`) with once-per-process deprecation warning, and runs `normalizeAndValidateStorageConfig` (auto-fixes missing trailing `/`, throws `StorageConfigError` on tier overlap). Path-segment matcher: `media/x/` does NOT match `media/xerox/foo`. Replaces gray-matter (broken on delimiter-less YAML) with a dedicated parser for the `gbrain.yml` shape. @@ -260,7 +260,7 @@ strict behavior when unset. - `admin/` (v0.26.0) — React 19 + Vite + TypeScript admin SPA embedded in the binary via `admin/dist/` served by `serve-http.ts`. 7 screens: Login (bootstrap token → session cookie), Dashboard (metrics + SSE feed + token health), Agents (sortable table + sparklines + Register button), Register (modal with scope checkboxes + grant type selector), Credentials reveal (full-screen modal with Copy + Download JSON + yellow one-time-only warning), Request Log (filterable paginated), Agent Detail drawer (Details / Activity / Config Export tabs + Revoke). Design tokens: `#0a0a0f` bg, Inter for UI, JetBrains Mono for data, 4-32px spacing scale, rounded pill badges. HTTP-only SameSite=Strict cookie auth. 65KB gzip. Build: `cd admin && bun install && bun run build`; output at `admin/dist/` is committed for self-contained binaries. - `src/commands/auth.ts` — Token management. `gbrain auth create/list/revoke/test` for legacy bearer tokens (v0.22.7 wired as a first-class CLI subcommand) plus `gbrain auth register-client` (v0.26.0) and `gbrain auth revoke-client ` (v0.26.2) for OAuth 2.1 client lifecycle. `revoke-client` runs an atomic `DELETE...RETURNING` on `oauth_clients`; FK `ON DELETE CASCADE` on `oauth_tokens.client_id` and `oauth_codes.client_id` purges every active token + authorization code in a single transaction. `process.exit(1)` on no-such-client (idempotent — re-running on the same id produces the same exit-1 message). Legacy tokens stored as SHA-256 hashes in `access_tokens`; OAuth clients in `oauth_clients`. As of v0.26.0, legacy tokens grandfather to `read+write+admin` scopes on the OAuth HTTP server, so pre-v0.26 deployments keep working with no migration. **v0.31.3 (#681):** every SQL site routes through `sqlQueryForEngine(engine)` from `src/core/sql-query.ts` (and `executeRawJsonb` for the takes-holders `permissions` JSONB column) so `gbrain auth` works against PGLite brains. Pre-fix, every call hit the postgres.js singleton via `getConn()` and silently failed (or wrote to the wrong DB) when the active engine was PGLite. The takes-holders write goes through `executeRawJsonb(engine, sql, [name, hash], [{takes_holders:[...]}])` which round-trips with `jsonb_typeof = 'object'` instead of the pre-v0.31.3 quoted-string shape. **v0.34.1.0 (#876):** `register-client` accepts `--source ` (write authority, scalar) and `--federated-read ` (read scope, array). The output prints the resolved `Write source` and `Federated reads` for the registered client. Pre-v0.34 clients backfill to `source_id='default'` via migration v60 so existing deployments keep their v0.33 effective behavior verbatim. - `src/commands/upgrade.ts` — Self-update CLI. `runPostUpgrade()` enumerates migrations from the TS registry (src/commands/migrations/index.ts) and tail-calls `runApplyMigrations(['--yes', '--non-interactive'])` so the mechanical side of every outstanding migration runs unconditionally. -- `src/commands/migrations/` — TS migration registry (compiled into the binary; no filesystem walk of `skills/migrations/*.md` needed at runtime). `index.ts` lists migrations in semver order. `v0_11_0.ts` = Minions adoption orchestrator (8 phases). `v0_12_0.ts` = Knowledge Graph auto-wire orchestrator (5 phases: schema → config check → backfill links → backfill timeline → verify). `phaseASchema` has a 600s timeout (bumped from 60s in v0.12.1 for duplicate-heavy brains). `v0_12_2.ts` = JSONB double-encode repair orchestrator (4 phases: schema → repair-jsonb → verify → record). `v0_14_0.ts` = shell-jobs + autopilot cooperative (2 phases: schema ALTER minion_jobs.max_stalled SET DEFAULT 3 — superseded by v0.14.3's schema-level DEFAULT 5 + UPDATE backfill; pending-host-work ping for skills/migrations/v0.14.0.md). All orchestrators are idempotent and resumable from `partial` status. As of v0.14.2 (Bug 3), the RUNNER owns all ledger writes — orchestrators return `OrchestratorResult` and `apply-migrations.ts` persists a canonical `{version, status, phases}` shape after return. Orchestrators no longer call `appendCompletedMigration` directly. `statusForVersion` prefers `complete` over `partial` (never regresses). 3 consecutive partials → wedged → `--force-retry ` writes a `'retry'` reset marker. v0.14.3 (fix wave) ships schema-only migrations v14 (`pages_updated_at_index`) + v15 (`minion_jobs_max_stalled_default_5` with UPDATE backfill) via the `MIGRATIONS` array in `src/core/migrate.ts` — no orchestrator phases needed. +- `src/commands/migrations/` — TS migration registry (compiled into the binary; no filesystem walk of `skills/migrations/*.md` needed at runtime). `index.ts` lists migrations in semver order. `v0_11_0.ts` = Minions adoption orchestrator (8 phases). `v0_12_0.ts` = Knowledge Graph auto-wire orchestrator (5 phases: schema → config check → backfill links → backfill timeline → verify). `phaseASchema` has a 600s timeout (bumped from 60s in v0.12.1 for duplicate-heavy brains). `v0_12_2.ts` = JSONB double-encode repair orchestrator (4 phases: schema → repair-jsonb → verify → record). `v0_14_0.ts` = shell-jobs + autopilot cooperative (2 phases: schema ALTER minion_jobs.max_stalled SET DEFAULT 3 — superseded by v0.14.3's schema-level DEFAULT 5 + UPDATE backfill; pending-host-work ping for skills/migrations/v0.14.0.md). All orchestrators are idempotent and resumable from `partial` status. As of v0.14.2 (Bug 3), the RUNNER owns all ledger writes — orchestrators return `OrchestratorResult` and `apply-migrations.ts` persists a canonical `{version, status, phases}` shape after return. Orchestrators no longer call `appendCompletedMigration` directly. `statusForVersion` prefers `complete` over `partial` (never regresses). 3 consecutive partials → wedged → `--force-retry ` writes a `'retry'` reset marker. v0.14.3 (fix wave) ships schema-only migrations v14 (`pages_updated_at_index`) + v15 (`minion_jobs_max_stalled_default_5` with UPDATE backfill) via the `MIGRATIONS` array in `src/core/migrate.ts` — no orchestrator phases needed. **v0.41.37.0 (#1605):** new `in-process.ts` exports `runMigrateOnlyCore({timeoutMs?})` — the single source of truth for "bring schema to head" (`configureGateway` → `createEngine` → `connect` → `initSchema` → `disconnect`, idempotent, 600s wall-clock guard `MIGRATE_ONLY_TIMEOUT_MS`, throws `MigrateOnlyError` on no-config / timeout). The migration orchestrators' 9 schema phases AND `init.ts:initMigrateOnly` (the `gbrain init --migrate-only` CLI path) both delegate to it, so the schema bring-up can't drift between them. Previously each phase shelled out to a child `gbrain init --migrate-only` via `execSync`; on Windows + bun + Supabase pooler the SPAWNED child died with `getaddrinfo ENOTFOUND` before it could connect (a bun-on-Windows child-process DNS failure, NOT an env-propagation bug — the parent connects fine with `env: process.env`). Running in-process removes the spawn entirely. `runGbrainSubprocess` is the diagnostic wrapper for the REMAINING non-schema spawns (extract/repair/stats): captures child stderr (64MB buffer) and folds it into the thrown error so a Windows failure shows the real `getaddrinfo ENOTFOUND` line instead of a bare `Command failed`. Pinned by `test/migration-in-process.serial.test.ts`. **v0.41.37.0 (#1581):** `v0_13_1.ts:phaseCGrandfather` rewritten from a per-page `getPage` + `putPage` loop (which hung CPU-bound 70+ min on an 82K-page PGLite brain) to a CHUNKED bulk SQL pass. Three correctness properties the per-page loop and a naive single bulk UPDATE both got wrong: (1) keyed on `pages.id` (globally unique PK), NOT slug — slug uniqueness is `(source_id, slug)`, so a slug-batched UPDATE would mutate same-slug pages across other sources; (2) filters `deleted_at IS NULL` so tombstones aren't grandfathered; (3) chunked in `CHUNK_SIZE` batches (DELETE_BATCH_SIZE convention) so lock-hold stays bounded instead of one giant transaction. The rollback log carries source identity (`{id, slug, source_id, pre_frontmatter}`) so a rollback is unambiguous across sources. Idempotent + resumable (each UPDATE flips its rows out of `GRANDFATHER_WHERE`). Completes ~1-2s on the 82K-page brain. Pinned by `test/migrations-v0_13_1-grandfather.test.ts`. - `src/commands/repair-jsonb.ts` — `gbrain repair-jsonb [--dry-run] [--json]`: rewrites `jsonb_typeof='string'` rows in place across 5 affected columns (pages.frontmatter, raw_data.data, ingest_log.pages_updated, files.metadata, page_versions.frontmatter). Fixes v0.12.0 double-encode bug on Postgres; PGLite no-ops. Idempotent. - `src/commands/orphans.ts` — `gbrain orphans [--json] [--count] [--include-pseudo] [--source ]`: surfaces pages with zero inbound wikilinks, grouped by domain. Auto-generated/raw/pseudo pages filtered by default. Also exposed as `find_orphans` MCP operation. Shipped in v0.12.3 (contributed by @knee5). **v0.41.29.0 source-scoping wave:** `findOrphans`/`getOrphansData` (the canonical pure fn shared with doctor's `orphan_ratio`) takes `{ sourceId?, sourceIds? }`; `BrainEngine.findOrphanPages(opts?)` (both engines) scopes ONLY the candidate side (`p.source_id = $1` scalar, or `= ANY($1::text[])` federated) while still counting inbound links from ANY source — a page in source X linked FROM source Y is reachable, so NOT an orphan of X (the deliberate, less-surprising definition; the stricter intra-source-only reading is rejected). `--source` is an explicit raw-flag parse (NOT `resolveSourceWithTier`, which would scope bare invocations to a default). Also corrected the `total_linkable` denominator: it now enumerates ALL live pages (scoped) and subtracts every excluded-by-slug page (templates/, scratch/, etc.) — pre-fix `total - excludedOrphans` left excluded NON-orphan pages with inbound links in the denominator, inflating it and suppressing warnings (changes orphan_ratio output for every brain, in the accurate direction). The `find_orphans` MCP op threads `sourceScopeOpts(ctx)` so a source-bound OAuth client no longer sees brain-wide orphans (closes a cross-source read leak in the v0.34.1 class). `gbrain doctor --source ` scopes `orphan_ratio` (entity-count gate + getOrphansData + messages) and, under explicit `--source` below 100 entity pages, reports the ratio with a low-scale caveat instead of a vacuous "ok". Thin-client `doctor --source` orphan_ratio remains brain-wide (TODO). Pinned by `test/orphans-source-scope.test.ts` (PGLite) + `test/e2e/engine-parity.test.ts` (Postgres↔PGLite scalar + federated parity). - `src/commands/salience.ts` (v0.29) — `gbrain salience [--days N] [--limit N] [--kind PREFIX] [--json]`: pages ranked by emotional + activity salience over a recency window. Mirrors orphans.ts shape (pure data fn + JSON formatter + human formatter). Calls `engine.getRecentSalience(opts)`. Score formula: `(emotional_weight × 5) + ln(1 + active_take_count) + 1/(1 + days_since_update)`. @@ -287,7 +287,7 @@ strict behavior when unset. - `src/core/embedding-dim-check.ts` extension (v0.41.17.0, T5+T6) — facts.embedding dim drift surface. New `readFactsEmbeddingDim(engine): Promise` covers both `vector(N)` and `halfvec(N)` shapes (codex #19 — migration v40 falls back to `vector` on pgvector < 0.7). Regex ordering halfvec-before-vector pinned by tests (substring "vec" appears in "halfvec"; naive `/vector/i` would shadow). New `buildFactsAlterRecipe(dims, configured, type)` emits the paste-ready `DROP INDEX IF EXISTS idx_facts_embedding_hnsw; ALTER TABLE facts ALTER COLUMN embedding TYPE halfvec(N) USING embedding::halfvec(N); CREATE INDEX idx_facts_embedding_hnsw ON facts USING hnsw (embedding halfvec_cosine_ops) WHERE ...` flow (codex #18 — NOT bare REINDEX which doesn't rewrite the index after a column-type change). New `assertFactsEmbeddingDimMatchesConfig(engine)` is the D15 preflight — throws `FactsEmbeddingDimMismatchError` (tagged class with `tag: 'FACTS_EMBEDDING_DIM_MISMATCH'` for parity with the worker-pool's MUST_ABORT semantics) when configured dim doesn't match the column width. Result cached per-engine via `WeakMap`. PGLite engines silently skip. New doctor check `facts_embedding_width_consistency` (registered in `runDoctor` after `embedding_width_consistency`) reuses the same helpers — surfaces drift with paste-ready ALTER recipe identical to the preflight error. Pinned by 18 cases in `test/embedding-dim-check-facts.test.ts`. - `src/core/postgres-engine.ts` extension (v0.41.17.0, T6, codex #20) — `insertFact` + `insertFacts` no longer hardcode `tx.unsafe(\`'${embedLit}'::vector\`)` for the embedding column. New `resolveFactsEmbeddingCast()` private method probes `pg_attribute` once per engine instance (cached in `_factsEmbeddingCastSuffix`) and returns `'::halfvec'` when migration v40 created the column as halfvec, else `'::vector'`. Both insert paths use the cached suffix so the cast matches the actual column type. Pre-fix all three insert sites hardcoded `::vector`; works on pgvector >= 0.7 via implicit auto-cast but fails on older pgvector. Test seam `__resetFactsEmbeddingCastCacheForTest()` clears per-engine cache. - `src/core/cycle.ts` + `src/core/cycle/extract-atoms.ts` + `src/core/cycle/synthesize-concepts.ts` + `src/commands/extract.ts` + `src/commands/doctor.ts` + `src/core/op-checkpoint.ts` extensions (v0.41.20.0 ops-fix-wave) — six daily-driver ops pains in one bisectable wave. (1) **Batch idempotency for `extract_atoms`:** new `atomsExistingForHashes(engine, sourceId, hashes[])` exported from `src/core/cycle/extract-atoms.ts` replaces the per-hash loop that did 7K individual queries at the start of every cycle on brains with conversation-transcript corpora (5-10 min silent overhead). One batched SQL roundtrip returns the set of `content_hash16` values already extracted as atoms for this source. Fail-open: SQL error logs to stderr and returns an empty set so extraction proceeds. Powered by migration v104 `pages_atom_source_hash_idx` (partial expression index on `frontmatter->>'source_hash'` for atom rows where `deleted_at IS NULL`; Postgres uses `CREATE INDEX CONCURRENTLY` with invalid-remnant pre-drop mirroring v97 `pages_dedup_partial_index`, PGLite uses plain `CREATE INDEX`). Without this index the batch idempotency check would seq-scan pages on big brains and defeat the perf win. (2) **Shorter cycle lock TTL + active in-phase refresh:** `LOCK_TTL_MINUTES` dropped from 30 → 5 in `src/core/cycle.ts`. New exported `buildYieldDuringPhase(lock, outer)` closure calls `lock.refresh()` AND any external hook on every fire; throttled to 30s inside each phase via `maybeYield`. Fires both inside the main work loop AND immediately after every `await chat(...)` LLM call so long Haiku/Sonnet calls don't sit past TTL. `LockHandle` and `buildYieldDuringPhase` exported for test seam access. `synthesize_concepts` no longer fires `yieldDuringPhase` per-concept-group — same hook, throttled via the new shared `maybeYield` helper. A crashed cycle releases its lock 6x faster while a healthy long-running cycle keeps the lock alive. Known residual under shorter TTL: a single `await chat()` past 5 min wallclock can expire the lock mid-await without the original phase noticing — `DbLockHandle.refresh()` throwing on 0 rows affected is filed as P2 follow-up TODO-OPS-2. (3) **Progress wiring through long phases:** new `progress?: ProgressReporter` opt on `ExtractAtomsOpts` and `SynthesizeConceptsOpts`. Cycle.ts passes its phase-level reporter down (NOT a child reporter — that would produce a path collision `cycle.extract_atoms.extract_atoms.work`). Phases only call `tick()` and `heartbeat()`; cycle.ts owns `start()` and `finish()`. You see `[cycle.extract_atoms] N (atoms_created)` ticks every ~1s during both long phases instead of "start" then silence for 10+ min. (4) **`by-mention` resume:** new `mentionsFingerprint({source, type, since, gazetteerHash})` in `src/core/op-checkpoint.ts`. The gazetteer hash is the load-bearing field — adding new entity pages mid-pause shifts the hash, gets a new fingerprint, and triggers a fresh scan against the new gazetteer instead of silently skipping previously-scanned pages. `gbrain extract links --by-mention` now resumes from where it died via the existing `op_checkpoints` framework with a `flushAndCheckpoint` ordering — links flush to the DB FIRST, page keys commit to the checkpoint SECOND, persist THIRD. A crash between `batch.push()` and flush leaves the page un-checkpointed so resume re-scans it. Persist cadence: every 1000 items OR every 30s, whichever first. Clean exit clears the checkpoint. `--dry-run` deliberately skips both load and write so it stays an inspection mode. (5) **`sync_consolidation` doctor check:** multi-source brains see a paste-ready `gbrain sync --all --parallel 4 --workers 4 --skip-failed` recommendation. Single-source brains get "not applicable." SQL errors return `warn` via the check's own try/catch — outer doctor catch isn't a safe assumption. Companion "Multi-source brains" recipe block in `skills/cron-scheduler/SKILL.md` documents the `sync --all` pattern as preferred over per-source entries. (6) **Test-isolation fixes:** `test/cycle-last-full-cycle-at.test.ts` + `test/schema-cli.test.ts` migrated to per-test `GBRAIN_HOME=tempdir` isolation. Pinned by 44 new unit/PGLite cases across 9 files: `test/cycle/extract-atoms-batch.test.ts` (5 — batch idempotency), `test/cycle/cycle-lock-ttl.test.ts` (1 — regression pin on `LOCK_TTL_MINUTES === 5`), `test/op-checkpoint-mentions-fingerprint.test.ts` (7 — fingerprint sensitivity including gazetteer-hash regression guard), `test/cycle/extract-atoms-progress.test.ts` (4 — phase doesn't call start/finish, ticks fire per item), `test/cycle/synthesize-concepts-progress.test.ts` (3 — same shape), `test/cycle/yield-during-phase-refresh.test.ts` (7 — buildYieldDuringPhase actually calls lock.refresh() + outer hook, throws non-fatal), `test/cycle/yield-during-phase-throttle.test.ts` (3 — 30s throttle gate), `test/extract-by-mention-resume.test.ts` (5 — checkpoint persistence ordering, dry-run skips persist, gazetteer change invalidates, filtered pages get checkpointed), `test/doctor-sync-consolidation.test.ts` (6 — edge case matrix for source counts + archived filtering + SQL error path). Two follow-up TODOs filed in TODOS.md under "v0.41.19.0 ops-fix-wave follow-ups": `gbrain sync print-cron` subcommand (TODO-OPS-1) and lock-loss detection via `DbLockHandle.refresh()` throwing on 0 rows affected (TODO-OPS-2). -- `src/commands/sync.ts` — `gbrain sync` CLI + the `performSync` / `performFullSync` library entrypoints (consumed by the autopilot cycle and the Minion sync handler). v0.22.13 (PR #490): `performSync` wraps its body in a `gbrain-sync` writer lock so two concurrent syncs (manual + autopilot, two terminals, two Conductor workspaces) cannot both write `last_commit` and let the last writer win. Head-drift gate after the import phase re-checks `git rev-parse HEAD`; if HEAD moved (someone ran `git checkout` / `git pull` mid-sync), the bookmark refuses to advance. Vanished files now record a failedFiles entry instead of silent-skip — the silent-skip-then-advance pathology that survived prior hardening passes is dead. Worker engines wrap in try/finally so disconnect always fires (panic-path leak fix). Both PGLite-detection sites use `engine.kind === 'pglite'`. CLI accepts `--workers N` (alias `--concurrency N`), validated via `parseWorkers`. Explicit `--workers` bypasses the auto-path file-count floor; auto path defers to `autoConcurrency()`. Banner moved to stderr. **v0.34.2.0:** the inline `.sort()` over add/mod paths is replaced with `sortNewestFirst(addsAndMods)` from `src/core/sort-newest-first.ts`, so the newest-first descending-lex policy lives in one helper shared with `gbrain import` instead of drifting across two files. **v0.40.3.0 (productionized from PR #1314 by @garrytan-agents):** the load-bearing change is the lock-identity invariant — `performSync` now defaults to a per-source lock id (`gbrain-sync:`) whenever `opts.sourceId` is set, AND wraps the writer window in `withRefreshingLock` from `src/core/db-lock.ts` so long-running sources (the PR motivator: media-corpus / 250K+ chunks) don't lose their lock at the 30-min TTL mid-run. Legacy single-default-source brains keep the bare `tryAcquireDbLock(SYNC_LOCK_ID)` path for back-compat. `SyncOpts.lockId?: string` is the explicit override (escape hatch; production callers don't set it). The fix closes the bug class Codex's outside-voice review caught in the original PR — a `sync --all` worker on per-source lock racing against `sync --source foo` on global lock would have silently corrupted the same source. `gbrain sync --all` got a continuous worker pool: `parseWorkers`-validated `--parallel N` (default `min(sourceCount, --workers, DEFAULT_PARALLEL_SOURCES=4)` from `src/core/sync-concurrency.ts`), `parallel` long-lived async workers pulling from a shared FIFO queue (no head-of-line blocking from wave dispatch), per-source `withSourcePrefix(src.id, ...)` wrap from `src/core/console-prefix.ts` so every `slog`/`serr` line carries `[] ` for kubectl-style greppable parallel output. `--skip-failed` and `--retry-failed` reject with a paste-ready error when combined with `--parallel > 1` (the brain-global `sync-failures.jsonl` has no per-source scope; v0.41+ TODO source-scopes it). Connection-budget stderr warning fires when `parallel × workers × 2 > 16` (the `× 2 per-file pool` factor: each per-file worker opens its own `PostgresEngine` with `poolSize=2`, so `--parallel 4 --workers 4` is actually 32 connections, not 16 — Codex caught the 2× understatement in the original plan). New exports `resolveParallelism`, `syncOneSource`, `buildSyncStatusReport`, `printSyncStatusReport`, `SyncStatusReport` back the `gbrain sources status` dashboard surface (D3 → `sources` subcommand, not a `sync` flag, so reads and writes don't share a verb). Stable `--json` envelope `{schema_version: 1, sources, parallel, ok_count, error_count}` on stdout under `--json`; human banners route to stderr via the `humanSink` helper so `jq` parses cleanly. Exit matrix: 0 = all ok, 1 = any error, 2 = cost-prompt-not-confirmed (unchanged). The dashboard SQL is the canonical `content_chunks ch JOIN pages pg ON pg.id = ch.page_id WHERE pg.deleted_at IS NULL` shape with `archived = false` filter at the caller — the original PR shipped `chunks ch JOIN ON page_slug`, which would have crashed on PGLite parse and silently zeroed on Postgres via the swallow-catch. Embedding column resolved via `resolveEmbeddingColumn(undefined, cfg)` from `src/core/search/embedding-column.ts` so Voyage / multimodal / non-default-column brains see counts against the column they actually use. Errors propagate from the dashboard SQL — no swallow-catch (Q2 sub-fix). The IRON RULE regression lives in `test/e2e/sync-status-pglite.test.ts`: real PGLite seeds 2 sources × pages × chunks, soft-deletes 1 page, archives 1 source, validates the SQL excludes both AND the active embedding column is the one used. 38 in-file `console.log`/`console.error` call sites inside `performSync*` migrated to `slog`/`serr` (top-level `runSync` orchestrator console calls intentionally stay outside the prefix scope). **v0.41.25.0 (supersedes PR #1538):** sync delete loop rewritten as interleaved per-batch resolve+delete using `engine.resolveSlugsByPaths` + `engine.deletePages` from `src/core/engine.ts`. 73K-delete commit drops from ~146K SQL round-trips (~5h on Postgres+pgbouncer) to ~292 round-trips (~2min) — closes the cascade-staleness bug class where one big-delete commit jammed every other source's sync for hours. Per-batch try-catch decomposes batch DELETE failures to per-slug `deletePage` fallback so a transient blip on batch 73 doesn't lose 500 deletes; unrecoverable per-slug failures land in `failedFiles` (matches the existing import-loop pattern). `pagesAffected` filters to D6 confirmed-deleted slugs (downstream extract/embed stop wasting work on phantoms). Same batched slug-resolve treatment for the rename loop (`gbrain sync` renames — Phase 3 in the plan; the per-file `importFile` stays per-file, only the slug-resolve N+1 gets batched). The `resolveSlugByPathOrSourcePath` single-call helper at `sync.ts:267` now delegates to `engine.resolveSlugsByPaths` when `sourceId` is set (D8 DRY — one owner of the SQL + fallback semantics), keeping legacy `executeRaw` fallback for the no-sourceId path which is functionally dead post-v0.34.1's source-resolution wiring. Legacy no-sourceId branch in the delete loop preserves correctness on a slow path; the engine batch methods require sourceId so the new fast path only fires when production threading is intact. `failedFiles` declaration hoisted to top of `performSyncInner` so both delete decompose and import loops feed the same sync-bookmark gate. **v0.41.31.0 (mode-aware cost gate):** the v0.20.0 unconditional `sync --all` ConfirmationRequired gate is superseded by a mode-aware gate driven by `willEmbedSynchronously` + `shouldBlockSync` from `src/core/embedding.ts`. federated_v2 is resolved ONCE up front (shared with the fan-out below). On the DEFERRED path (v2 on, parallel) embedding goes to per-source `embed-backfill` jobs that carry their own `$X/source/24h` cap (default $25, `SPEND_CAP_CONFIG_KEY` from `embed-backfill-submit.ts`), so the gate is INFORMATIONAL — it prints an FYI naming the backfill cap + current backlog (`sumStaleChunkChars({signature})` priced via `estimateCostFromChars` at `currentEmbeddingPricePerMTok()`) + the count of already-queued `embed-backfill` jobs (TODO-2 visibility), and NEVER exits 2. On the INLINE path (v2 off, or `--serial` without `--no-embed`) the BLOCKING gate fires only when the NEW-CONTENT estimate exceeds the `sync.cost_gate_min_usd` floor (default $0.50). The new-content estimate (`estimateInlineNewTokens`) contributes ZERO for sources provably unchanged since last sync (`isSourceUnchangedSinceSync` HEAD==last_commit + clean tree, AND `chunker_version === CURRENT`) — mirrors doctor's `sync_freshness` + sync's own do-work gate — and the full-tree ceiling for changed sources. F2 fix: the pre-existing stale backlog (NULL embeddings + signature drift) is shown informationally but NEVER gated on, because sync doesn't sweep it inline (`gbrain embed --stale` does); gating on it would block every inline cron after a model swap. New helpers `resolveCostGateFloorUsd(engine)` (reads `sync.cost_gate_min_usd`, fail-open $0.50, accepts 0) + `resolveBackfillCapUsd(engine)`. The `sync --all` SELECT widens to carry `last_commit + chunker_version` (both columns predate v0.41; no migration). JSON envelopes gain `mode` + `gate` discriminators (`dry_run | deferred_notice | below_floor | confirmation_required`). `SyncStatusReportSource` gains `backfill_queued` / `backfill_active` / `backfill_last_completed_at` (best-effort; 0/null on pre-minions brains). All embedding-cost preview math reads the configured model name via `getEmbeddingModelName()` (no hardcoded OpenAI). Pinned by `test/sync-cost-gate.serial.test.ts` + `test/sync-cost-preview.test.ts`. +- `src/commands/sync.ts` — `gbrain sync` CLI + the `performSync` / `performFullSync` library entrypoints (consumed by the autopilot cycle and the Minion sync handler). v0.22.13 (PR #490): `performSync` wraps its body in a `gbrain-sync` writer lock so two concurrent syncs (manual + autopilot, two terminals, two Conductor workspaces) cannot both write `last_commit` and let the last writer win. Head-drift gate after the import phase re-checks `git rev-parse HEAD`; if HEAD moved (someone ran `git checkout` / `git pull` mid-sync), the bookmark refuses to advance. Vanished files now record a failedFiles entry instead of silent-skip — the silent-skip-then-advance pathology that survived prior hardening passes is dead. Worker engines wrap in try/finally so disconnect always fires (panic-path leak fix). Both PGLite-detection sites use `engine.kind === 'pglite'`. CLI accepts `--workers N` (alias `--concurrency N`), validated via `parseWorkers`. Explicit `--workers` bypasses the auto-path file-count floor; auto path defers to `autoConcurrency()`. Banner moved to stderr. **v0.34.2.0:** the inline `.sort()` over add/mod paths is replaced with `sortNewestFirst(addsAndMods)` from `src/core/sort-newest-first.ts`, so the newest-first descending-lex policy lives in one helper shared with `gbrain import` instead of drifting across two files. **v0.40.3.0 (productionized from PR #1314 by @garrytan-agents):** the load-bearing change is the lock-identity invariant — `performSync` now defaults to a per-source lock id (`gbrain-sync:`) whenever `opts.sourceId` is set, AND wraps the writer window in `withRefreshingLock` from `src/core/db-lock.ts` so long-running sources (the PR motivator: media-corpus / 250K+ chunks) don't lose their lock at the 30-min TTL mid-run. Legacy single-default-source brains keep the bare `tryAcquireDbLock(SYNC_LOCK_ID)` path for back-compat. `SyncOpts.lockId?: string` is the explicit override (escape hatch; production callers don't set it). The fix closes the bug class Codex's outside-voice review caught in the original PR — a `sync --all` worker on per-source lock racing against `sync --source foo` on global lock would have silently corrupted the same source. `gbrain sync --all` got a continuous worker pool: `parseWorkers`-validated `--parallel N` (default `min(sourceCount, --workers, DEFAULT_PARALLEL_SOURCES=4)` from `src/core/sync-concurrency.ts`), `parallel` long-lived async workers pulling from a shared FIFO queue (no head-of-line blocking from wave dispatch), per-source `withSourcePrefix(src.id, ...)` wrap from `src/core/console-prefix.ts` so every `slog`/`serr` line carries `[] ` for kubectl-style greppable parallel output. `--skip-failed` and `--retry-failed` reject with a paste-ready error when combined with `--parallel > 1` (the brain-global `sync-failures.jsonl` has no per-source scope; v0.41+ TODO source-scopes it). Connection-budget stderr warning fires when `parallel × workers × 2 > 16` (the `× 2 per-file pool` factor: each per-file worker opens its own `PostgresEngine` with `poolSize=2`, so `--parallel 4 --workers 4` is actually 32 connections, not 16 — Codex caught the 2× understatement in the original plan). New exports `resolveParallelism`, `syncOneSource`, `buildSyncStatusReport`, `printSyncStatusReport`, `SyncStatusReport` back the `gbrain sources status` dashboard surface (D3 → `sources` subcommand, not a `sync` flag, so reads and writes don't share a verb). Stable `--json` envelope `{schema_version: 1, sources, parallel, ok_count, error_count}` on stdout under `--json`; human banners route to stderr via the `humanSink` helper so `jq` parses cleanly. Exit matrix: 0 = all ok, 1 = any error, 2 = cost-prompt-not-confirmed (unchanged). The dashboard SQL is the canonical `content_chunks ch JOIN pages pg ON pg.id = ch.page_id WHERE pg.deleted_at IS NULL` shape with `archived = false` filter at the caller — the original PR shipped `chunks ch JOIN ON page_slug`, which would have crashed on PGLite parse and silently zeroed on Postgres via the swallow-catch. Embedding column resolved via `resolveEmbeddingColumn(undefined, cfg)` from `src/core/search/embedding-column.ts` so Voyage / multimodal / non-default-column brains see counts against the column they actually use. Errors propagate from the dashboard SQL — no swallow-catch (Q2 sub-fix). The IRON RULE regression lives in `test/e2e/sync-status-pglite.test.ts`: real PGLite seeds 2 sources × pages × chunks, soft-deletes 1 page, archives 1 source, validates the SQL excludes both AND the active embedding column is the one used. 38 in-file `console.log`/`console.error` call sites inside `performSync*` migrated to `slog`/`serr` (top-level `runSync` orchestrator console calls intentionally stay outside the prefix scope). **v0.41.25.0 (supersedes PR #1538):** sync delete loop rewritten as interleaved per-batch resolve+delete using `engine.resolveSlugsByPaths` + `engine.deletePages` from `src/core/engine.ts`. 73K-delete commit drops from ~146K SQL round-trips (~5h on Postgres+pgbouncer) to ~292 round-trips (~2min) — closes the cascade-staleness bug class where one big-delete commit jammed every other source's sync for hours. Per-batch try-catch decomposes batch DELETE failures to per-slug `deletePage` fallback so a transient blip on batch 73 doesn't lose 500 deletes; unrecoverable per-slug failures land in `failedFiles` (matches the existing import-loop pattern). `pagesAffected` filters to D6 confirmed-deleted slugs (downstream extract/embed stop wasting work on phantoms). Same batched slug-resolve treatment for the rename loop (`gbrain sync` renames — Phase 3 in the plan; the per-file `importFile` stays per-file, only the slug-resolve N+1 gets batched). The `resolveSlugByPathOrSourcePath` single-call helper at `sync.ts:267` now delegates to `engine.resolveSlugsByPaths` when `sourceId` is set (D8 DRY — one owner of the SQL + fallback semantics), keeping legacy `executeRaw` fallback for the no-sourceId path which is functionally dead post-v0.34.1's source-resolution wiring. Legacy no-sourceId branch in the delete loop preserves correctness on a slow path; the engine batch methods require sourceId so the new fast path only fires when production threading is intact. `failedFiles` declaration hoisted to top of `performSyncInner` so both delete decompose and import loops feed the same sync-bookmark gate. **v0.41.31.0 (mode-aware cost gate):** the v0.20.0 unconditional `sync --all` ConfirmationRequired gate is superseded by a mode-aware gate driven by `willEmbedSynchronously` + `shouldBlockSync` from `src/core/embedding.ts`. federated_v2 is resolved ONCE up front (shared with the fan-out below). On the DEFERRED path (v2 on, parallel) embedding goes to per-source `embed-backfill` jobs that carry their own `$X/source/24h` cap (default $25, `SPEND_CAP_CONFIG_KEY` from `embed-backfill-submit.ts`), so the gate is INFORMATIONAL — it prints an FYI naming the backfill cap + current backlog (`sumStaleChunkChars({signature})` priced via `estimateCostFromChars` at `currentEmbeddingPricePerMTok()`) + the count of already-queued `embed-backfill` jobs (TODO-2 visibility), and NEVER exits 2. On the INLINE path (v2 off, or `--serial` without `--no-embed`) the BLOCKING gate fires only when the NEW-CONTENT estimate exceeds the `sync.cost_gate_min_usd` floor (default $0.50). The new-content estimate (`estimateInlineNewTokens`) contributes ZERO for sources provably unchanged since last sync (`isSourceUnchangedSinceSync` HEAD==last_commit + clean tree, AND `chunker_version === CURRENT`) — mirrors doctor's `sync_freshness` + sync's own do-work gate — and the full-tree ceiling for changed sources. F2 fix: the pre-existing stale backlog (NULL embeddings + signature drift) is shown informationally but NEVER gated on, because sync doesn't sweep it inline (`gbrain embed --stale` does); gating on it would block every inline cron after a model swap. New helpers `resolveCostGateFloorUsd(engine)` (reads `sync.cost_gate_min_usd`, fail-open $0.50, accepts 0) + `resolveBackfillCapUsd(engine)`. The `sync --all` SELECT widens to carry `last_commit + chunker_version` (both columns predate v0.41; no migration). JSON envelopes gain `mode` + `gate` discriminators (`dry_run | deferred_notice | below_floor | confirmation_required`). `SyncStatusReportSource` gains `backfill_queued` / `backfill_active` / `backfill_last_completed_at` (best-effort; 0/null on pre-minions brains). All embedding-cost preview math reads the configured model name via `getEmbeddingModelName()` (no hardcoded OpenAI). Pinned by `test/sync-cost-gate.serial.test.ts` + `test/sync-cost-preview.test.ts`. **v0.41.37.0 (#1569):** new `SyncOpts.noSchemaPack` (CLI `--no-schema-pack`, threaded through `performSync` AND `syncOneSource` so `sync --all` honors it) — skips `loadActivePack` entirely so no user-supplied pack page-type regex runs during import; pages fall back to legacy prefix typing. Escape hatch for completing a sync when a suspect pack regex is the suspected cause of a wedge (re-run extraction later). Same release adds a per-file BEGIN heartbeat: `if (process.env.GBRAIN_SYNC_TRACE) serr('[sync] begin import: ')` emitted BEFORE `importFile` (the existing `progress.tick` fires only AFTER `importFile` returns — useless when one file wedges). Off by default to avoid a line per file on huge brains; `serr` is source-prefix-aware so under `--workers >1` / `--all` the stuck file is the begin-line with no matching completion. Triage doc: `docs/architecture/serve-sync-concurrency.md` (PGLite single-writer serve↔sync contention — stop `gbrain serve` before a large PGLite sync — plus the `GBRAIN_SYNC_TRACE` + `--no-schema-pack` hang-triage recipes). - `src/commands/import.ts` — `gbrain import` CLI + `runImport` library entrypoint. v0.34.2.0 replaces the prior positional-index checkpoint (`processedIndex: N` into a sorted file list) with a path-set checkpoint via `src/core/import-checkpoint.ts`. The walk still applies `sortNewestFirst()` for embed-cost ordering, but checkpoint correctness no longer depends on sort order. A file enters `completed: Set` only when its `processFile` returns success (including content-hash short-circuit no-ops); failed files never enter the set, so the next run retries them automatically with no manual `~/.gbrain/import-checkpoint.json` delete. Three bug classes died: parallel-import-with-slow-worker drops the slow file on crash-resume (closed — the slow file isn't in `completed` until its own `processFile` resolves), failed-file-bumps-counter-past-itself (closed — failures don't add to `completed`), and v0.33.x sort-flip-drops-newest-N-on-cross-version-resume (closed — order is no longer part of the checkpoint). Old positional checkpoints are detected and discarded with a stderr line on first resume; re-walking is cheap because `content_hash` short-circuits unchanged files. Checkpoint persists every 100 successful adds, not every 100 processed files, so a long failure tail doesn't churn the JSON. Pinned by `test/import-checkpoint.test.ts` (18 unit cases over the helpers) + `test/import-resume.test.ts` (5 integration cases under PGLite, including the SLUG_MISMATCH retry regression codex caught during plan-eng-review). - `src/core/import-checkpoint.ts` (v0.34.2.0) — `loadCheckpoint(brainDir)`, `saveCheckpoint(brainDir, completed)`, `resumeFilter(files, completed, brainDir)`, `clearCheckpoint()`, plus the `ImportCheckpoint` type. Path-set checkpoint format (`{schema_version, brainDir, completed: string[]}`) replaces the v0.33.x positional `{processedIndex: N}` format. Atomic write via `.tmp` + `rename()` so a mid-write crash never leaves a partial JSON. `loadCheckpoint` returns `null` on: missing file, malformed JSON, brainDir mismatch (you ran import against a different brain), and the old positional format (logged to stderr before being discarded). `resumeFilter` returns `{toProcess, skippedCount}` — pure, no I/O, deterministic. `clearCheckpoint` is no-op-on-missing for clean-exit cleanup. Honors `GBRAIN_HOME` via `gbrainPath()` so test isolation via `withEnv({GBRAIN_HOME: tmpdir})` works without monkey-patching the fs layer. Best-effort persistence — `saveCheckpoint` logs warnings on write errors but never throws, so import keeps making progress even if disk is full. - `src/core/sort-newest-first.ts` (v0.34.2.0) — single source of truth for the descending-lex sort that `gbrain import` and `gbrain sync` both apply. Mutates in place (Array.prototype.sort semantics), returns the same array reference for fluent chaining. Empty/single-element inputs short-circuit. Future ordering changes flip one line in this helper instead of touching two CLI commands. Pinned by `test/sort-newest-first.test.ts` (5 hermetic cases: descending order, mixed prefixes, empty input, single-element input, in-place-mutation contract). @@ -623,7 +623,8 @@ Key files (v0.40.7.0 additions): - `src/core/schema-pack/mutate-audit.ts` — ISO-week JSONL at `~/.gbrain/audit/schema-mutations-YYYY-Www.jsonl`. Privacy-redacted per D20: type names → sha8, prefixes → first slug segment only, matches `candidate-audit.ts` privacy posture. Logs BOTH success AND failure events so the v0.40.7+ `schema_pack_writability` doctor check has signal to read. `summarizeMutations()` is the cross-surface parity primitive. - `src/core/schema-pack/registry.ts` extensions — `invalidatePackCache(name?)` walks the extends-chain reverse-graph (codex C6 fix; pre-v0.40.7, editing a parent pack silently left children stale). `tryCachedPack(name)` TTL-gated fast path: inside `STAT_TTL_MS` (default 1000ms, env `GBRAIN_PACK_STAT_TTL_MS`) returns cached without statting. Outside the window: stats every file in the chain; cascade-invalidates on mtime change (D11 cross-process detection). - `src/core/schema-pack/best-effort.ts` — `loadActivePackBestEffort(ctx)` returns `ResolvedPack | null`. Single source of truth for the T1.5 wiring sites. `null` means EMPTY FILTER (NOT hardcoded defaults — D4 contract closing the silent-violation bug class). -- `src/core/schema-pack/lint-rules.ts` — 11 pure rule functions. `withMutation`'s pre-write validation gate composes the 9 file-plane rules; the 2 DB-aware rules (`extractable_empty_corpus`, `mutation_count_anomaly`) need an engine. Single source of truth consumed by CLI lint + MCP `schema_lint` + the pre-write validation gate. +- `src/core/schema-pack/lint-rules.ts` — 12 pure rule functions. `withMutation`'s pre-write validation gate composes the 10 file-plane rules; the 2 DB-aware rules (`extractable_empty_corpus`, `mutation_count_anomaly`) need an engine. Single source of truth consumed by CLI lint + MCP `schema_lint` + the pre-write validation gate. **v0.41.37.0 (#1569):** new file-plane rule `link_regex_catastrophic_backtrack` — advisory ReDoS pre-screen that flags the classic nested-quantifier shapes (`(a+)+`, `(a*)*`, `(a+)*`, `(\w+)+`) in a link_type's `inference.regex` via `NESTED_QUANTIFIER_RE`. WARNING not error: a hard reject would disable the whole pack on upgrade (pages fall back to legacy typing). The runtime input-length cap in `redos-guard.ts` is the actual safety net; this rule tells the pack author to fix the pattern. +- `src/core/schema-pack/redos-guard.ts` + `src/core/schema-pack/link-inference.ts` (v0.41.37.0, #1569) — ReDoS hardening for pack inference regexes. `redos-guard.ts` adds `MAX_REGEX_INPUT_CHARS` (default 64_000, env override `GBRAIN_MAX_REGEX_INPUT_CHARS`) — a hard input-length cap, the real runtime safety net (catastrophic backtracking needs a long input to blow up; a link-extraction `context` is normally a sentence or short paragraph). Over the cap, `runRegexBounded` throws the new tagged `RegexInputTooLargeError` and the regex is skipped (degrade-to-mentions) without entering the `node:vm`. `link-inference.ts:inferLinkTypeFromPack` no-budget branch (test contexts) previously ran `new RegExp(pattern).test(context)` UNBOUNDED — the one ReDoS hole with no timeout; it now routes through `runRegexBounded` so the input-length cap + per-regex vm timeout (`PER_REGEX_TIMEOUT_MS = 50`) apply on every path. Defensive hardening + diagnostics; the deterministic ~3100-file sync-wedge root cause remains open (no repro). Pinned by `test/redos-hardening.test.ts` + `test/schema-pack-lint-rules.test.ts`. - `src/core/schema-pack/query-cache-invalidator.ts` — `invalidateQueryCache(engine, sourceId?)` DELETEs query_cache rows so cached search results bound to old page types don't survive a schema mutation. Codex C9 fix. - `src/core/schema-pack/mutate.ts` — 8-step `withMutation` skeleton (bundled-guard → lock → read → mutator → validate → atomic write → audit → invalidate). 11 mutation primitives: `addTypeToPack`, `removeTypeFromPack` (with codex C14 reference check), `updateTypeOnPack`, `addAliasToType`, `removeAliasFromType`, `addPrefixToType`, `removePrefixFromType`, `addLinkTypeToPack`, `removeLinkTypeFromPack`, `setExtractableOnType`, `setExpertRoutingOnType`. Atomic write via `.tmp + fsync + rename` — pack file on disk is NEVER partial. Inline minimal JSON→YAML emitter so YAML packs stay YAML (does NOT preserve comments — pin pack.json if you care about layout). - `src/core/schema-pack/stats.ts` — `runStatsCore(engine, opts)` returns per-source + aggregate page counts + coverage % + `dead_prefixes` (declared prefixes with zero matching pages — agent's drilldown signal). Multi-source aware (`sourceIds[]` federated, `sourceId` single, or whole-brain). PGLite + Postgres parity via `executeRaw`. Empty brain → coverage:1.0 (vacuous truth). diff --git a/README.md b/README.md index 9c64ee8ab..3f7a5b67a 100644 --- a/README.md +++ b/README.md @@ -340,6 +340,53 @@ anthropic/claude-sonnet-4-6 --max-cost 5` failed with matched the colon form. Both shapes work now. No config change, no schema migration — `gbrain upgrade` is the whole fix. +**`gbrain reindex --markdown` wiped your auto/dream/signal-detector +tags?** v0.41.37.0 makes tag reconciliation add-only. Re-import and +`reindex --markdown` now ADD current frontmatter tags and never delete, +so enrichment tags written to the DB (auto-tag, dream synthesize, +signal-detector) survive a re-chunk. The reindex DB-only fallback also +reconstructs the full markdown (frontmatter + body + timeline) before +re-chunking, so a page with no on-disk source keeps its frontmatter, +title, and timeline instead of getting overwritten with empty +frontmatter. Trade-off: removing a tag from a page's frontmatter no +longer removes it from the DB on the next sync (frontmatter-tag removal +needs a provenance column, deferred). (Closes #1621.) + +**`gbrain sync` wedges on a large brain (no progress, high CPU)?** +v0.41.37.0 ships three things. First, name the stalling file: + +```bash +GBRAIN_SYNC_TRACE=1 gbrain sync --no-pull --no-embed --yes +``` + +The last `[sync] begin import: ` line with no following completion +is the file being processed when the hang hit. Second, if you suspect a +schema-pack `inference.regex` with catastrophic backtracking, complete +the sync with the pack disabled and re-run extraction later: + +```bash +gbrain sync --no-schema-pack --no-pull --no-embed --yes +``` + +`gbrain schema lint` now warns on the classic nested-quantifier ReDoS +shapes (`(a+)+`, `(a*)*`, …) in pack regexes, and the runtime caps +inference-regex input length (override via `GBRAIN_MAX_REGEX_INPUT_CHARS`). +Third, on a PGLite brain, stop `gbrain serve` before a large sync — +PGLite is single-writer and a live MCP server contends for the write +lock. See [`docs/architecture/serve-sync-concurrency.md`](docs/architecture/serve-sync-concurrency.md) +for the full triage. (Closes #1569.) + +**`gbrain init --migrate-only` / a schema migration fails on Windows +with `getaddrinfo ENOTFOUND`?** v0.41.37.0 runs the 9 schema-bring-up +phases in-process instead of spawning a child `gbrain init +--migrate-only` per phase. The spawned child died on +Windows + bun + Supabase pooler with a DNS-resolution failure even +though the parent connected fine; running in-process removes the spawn +entirely. The v0.13.1 grandfather migration that hung 70+ minutes on an +82K-page PGLite brain is also fixed — it now runs as a chunked bulk SQL +pass (keyed on the page PK, soft-delete-filtered, source-safe) that +completes in ~1-2 seconds. (Closes #1605, #1581.) + ## Docs - [`docs/INSTALL.md`](docs/INSTALL.md) — every install path, end to end diff --git a/TODOS.md b/TODOS.md index 9173c37a4..6a3a87900 100644 --- a/TODOS.md +++ b/TODOS.md @@ -1,5 +1,17 @@ # TODOS +## v0.41.37.0 critical-fix-wave follow-ups (v0.42+) + +Filed from the v0.41.37.0 wave (#1621 tag-wipe, #1581 grandfather hang, +#1605 Windows migration spawn, #1569 sync ReDoS hardening). Each item was +deliberately scoped out of the wave (see plan + GSTACK REVIEW REPORT at +`~/.claude/plans/system-instruction-you-are-working-greedy-quiche.md`). + +- [ ] **#1621-followup: tag_source provenance column for frontmatter-tag REMOVAL.** The wave shipped ADD-ONLY tag reconciliation (`src/core/import-file.ts`) — re-import never deletes tags, so DB-side enrichment tags survive. Trade-off: removing a tag from a page's frontmatter no longer removes it from the DB. To restore removal-on-edit without wiping enrichment tags, add a `tags.tag_source` column (migration, both engines), stamp `'frontmatter'` on import-path tags, and reconcile by deleting only `tag_source='frontmatter'` tags absent from the new frontmatter (enrichment/backfilled tags default NULL = preserved, so no enrichment-write-site enumeration needed). Priority: P3 (additive-metadata staleness is low-harm). + +- [ ] **#1605-followup: convert migration backfill-phase spawns to in-process.** v0.41.37.0 made the 9 schema phases (`gbrain init --migrate-only`) run in-process via `runMigrateOnlyCore`, which unblocks `schema_version` advancement on Windows+bun+Supabase. The remaining non-schema spawns (`extract links/timeline`, `repair-jsonb`) still shell out via `runGbrainSubprocess` — they now surface child stderr (so a Windows failure is diagnosable) but still fail on Windows. Convert them to in-process calls (the extract/repair command functions are callable with an engine) so Windows brains complete data backfill, not just schema. Sites: `src/commands/migrations/v0_12_0.ts` (extract), `v0_12_2.ts` (repair), `v0_13_0.ts` (extract). Priority: P2. + +- [ ] **#1569-followup: root-cause the 56K-file sync wedge with the reporter's repro.** v0.41.37.0 shipped ReDoS hardening (input-length cap + star-height lint + `--no-schema-pack` escape) + diagnostics (`GBRAIN_SYNC_TRACE=1` begin-heartbeat + PGLite serve/sync concurrency doc), but did NOT root-cause the deterministic wedge at ~3100 files — the reporter's redos-guard hypothesis didn't hold (it's not on the sync path). Get the reporter's sample files (`/tmp/gbrain-hang-sample.txt`, `/tmp/gbrain-prewedge-sample.txt`), reproduce, and pin the resume-mode deep-recursion pre-import phase (prime suspect: the walk/diff/checkpoint path). Priority: P1 once a repro exists; tracked on the #1569 thread. ## MCP skillpack distribution — PR2 (v0.41.37+) Filed from the v0.41.36.0 skill-catalog wave (`list_skills` / `get_skill`). diff --git a/VERSION b/VERSION index 4c1a4ec65..cafecd08a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.41.36.0 +0.41.37.0 diff --git a/docs/architecture/serve-sync-concurrency.md b/docs/architecture/serve-sync-concurrency.md new file mode 100644 index 000000000..567fa6449 --- /dev/null +++ b/docs/architecture/serve-sync-concurrency.md @@ -0,0 +1,54 @@ +# `gbrain serve` ↔ `gbrain sync` concurrency (PGLite) + +**Short version: on a PGLite brain, stop `gbrain serve` before a large sync.** + +## Why + +PGLite is a single-writer embedded Postgres (WASM). A running `gbrain serve` +(stdio or HTTP MCP) holds an open PGLite connection on the brain's data +directory. `gbrain sync` needs to write to that same data directory. The two +contend for PGLite's single-writer connection / write-lock — **this is NOT the +`gbrain-sync` advisory lock** (that's a separate, DB-row coordination lock for +two concurrent *syncs*). Confusing the two sends you debugging the wrong surface. + +Symptoms of serve↔sync contention on PGLite: + +- `gbrain sync` blocks acquiring the PGLite write lock, or makes very slow + progress, while a `gbrain serve` process is alive on the same brain. +- Killing stale `gbrain serve` MCP processes frees the lock and sync proceeds. + +## What to do + +1. Stop any `gbrain serve` process for this brain before a large sync: + ```bash + pkill -f 'gbrain serve' # or stop your MCP client / Claude Desktop / Cursor + gbrain sync --no-pull --no-embed --yes + ``` +2. Restart `gbrain serve` after the sync completes. + +This contention does **not** apply to the Postgres engine — Postgres tolerates +concurrent connections, so `serve` and `sync` can run simultaneously there. + +## Diagnosing a sync hang + +If a sync wedges (no progress, high CPU), re-run with the per-file begin trace +so the stalling file is named: + +```bash +GBRAIN_SYNC_TRACE=1 gbrain sync --no-pull --no-embed --yes +``` + +The last `[sync] begin import: ` line with no following completion is the +file being processed when the hang occurred. Under `--workers >1` / `--all`, +the stuck file is in the set of begin-lines without a matching completion. + +If you suspect a schema-pack regex is the cause (a pack with a +catastrophic-backtracking `inference.regex`), complete the sync with the pack +disabled and re-run extraction afterward: + +```bash +gbrain sync --no-schema-pack --no-pull --no-embed --yes +``` + +`gbrain schema lint` flags the classic nested-quantifier ReDoS shapes +(`(a+)+`, `(a*)*`, …) in pack regexes as warnings. diff --git a/llms-full.txt b/llms-full.txt index 16c2f0cc7..2c801acb7 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -200,13 +200,13 @@ strict behavior when unset. - `src/core/audit-slug-fallback.ts` (v0.32.7 CJK wave) — Weekly ISO-week-rotated audit JSONL at `~/.gbrain/audit/slug-fallback-YYYY-Www.jsonl`. `logSlugFallback(slug, sourcePath)` fires when `importFromFile` falls back to a frontmatter slug because `slugifyPath` returned empty (emoji / Thai / Arabic / non-CJK exotic-script filenames). `readRecentSlugFallbacks(days)` reads the last N days for `gbrain doctor`'s `slug_fallback_audit` check. Honors `GBRAIN_AUDIT_DIR` via the shared `resolveAuditDir()` from shell-audit.ts. Separate surface from `sync-failures.jsonl` per codex outside-voice review — that file carries bookmark-gating semantics that info events shouldn't trigger. - `src/core/embedding-pricing.ts` (v0.32.7 CJK wave) — `EMBEDDING_PRICING` map keyed `provider:model` for the post-upgrade reindex cost estimate. Sibling to `anthropic-pricing.ts`. Entries: OpenAI text-embedding-3-large ($0.13/1M), 3-small ($0.02/1M), ada-002 ($0.10/1M), Voyage 3-large ($0.18/1M), 3 ($0.06/1M). `lookupEmbeddingPrice(modelString)` returns a tagged union (`known` with price + `unknown` with provider name); `estimateCostFromChars(charCount, pricePerMTok)` uses 3.5 chars/token approximation. Unknown providers degrade gracefully to "estimate unavailable" instead of fabricating numbers. - `src/core/post-upgrade-reembed.ts` (v0.32.7 CJK wave) — Pure functions backing the `gbrain upgrade` chunker-bump cost prompt. `computeReembedEstimate(engine, model)` queries real SQL (`COUNT(*)` + `COALESCE(SUM(LENGTH(compiled_truth)) + SUM(LENGTH(timeline)), 0)`) on `pages WHERE chunker_version < MARKDOWN_CHUNKER_VERSION`. `formatReembedPrompt(est, graceSeconds)` is the stderr-line formatter. `runPostUpgradeReembedPrompt(engine, model, opts)` orchestrates the 10-second Ctrl-C window; TTY-only wait (non-TTY auto-proceeds for CI / cron); `GBRAIN_NO_REEMBED=1` bails out with a doctor-warning marker; `GBRAIN_REEMBED_GRACE_SECONDS=0` skips the wait. -- `src/commands/reindex.ts` (v0.32.7 CJK wave) — `gbrain reindex --markdown [--limit N] [--dry-run] [--json] [--no-embed] [--repo PATH]`. Walks `pages WHERE page_kind = 'markdown' AND chunker_version < MARKDOWN_CHUNKER_VERSION` in 100-row batches, ordered by id. Rows with non-null `source_path` re-import via `importFromFile`; rows without fall back to `importFromContent` against the stored `compiled_truth`. **Both paths pass `forceRechunk: true`** to bypass `importFromContent`'s `content_hash` short-circuit — without that flag (codex post-merge F1), the chunker version bump never reaches pages whose source content hasn't changed since last sync, AND master's v0.32.2 stripFactsFence privacy strip never applies to pre-strip chunks. Idempotent — partial-completion re-runs pick up where they left off via id-ordered batches. Wired into `src/commands/upgrade.ts:runPostUpgrade` after `apply-migrations`. +- `src/commands/reindex.ts` (v0.32.7 CJK wave) — `gbrain reindex --markdown [--limit N] [--dry-run] [--json] [--no-embed] [--repo PATH]`. Walks `pages WHERE page_kind = 'markdown' AND chunker_version < MARKDOWN_CHUNKER_VERSION` in 100-row batches, ordered by id. Rows with non-null `source_path` re-import via `importFromFile`; rows without fall back to `importFromContent` against the stored `compiled_truth`. **Both paths pass `forceRechunk: true`** to bypass `importFromContent`'s `content_hash` short-circuit — without that flag (codex post-merge F1), the chunker version bump never reaches pages whose source content hasn't changed since last sync, AND master's v0.32.2 stripFactsFence privacy strip never applies to pre-strip chunks. Idempotent — partial-completion re-runs pick up where they left off via id-ordered batches. Wired into `src/commands/upgrade.ts:runPostUpgrade` after `apply-migrations`. **v0.41.37.0 (#1621):** the DB-only fallback (no source file on disk) no longer passes body-only `compiled_truth` to `importFromContent` — that path re-parsed with EMPTY frontmatter and OVERWROTE the page's real frontmatter / title / timeline (codex catch). It now `getPage` + `getTags`, reconstructs the FULL markdown via `serializeMarkdown(frontmatter, compiled_truth, timeline, {type, title, tags})`, and re-imports THAT, so re-chunking a DB-only page preserves everything while still bumping `chunker_version`. Pinned by `test/reindex-preserve-tags.test.ts`. - `src/commands/reindex-code.ts` (v0.21.0 Cathedral II E2, extended v0.37.3.0) — `gbrain reindex --code [--source ID] [--dry-run] [--yes] [--json] [--force] [--no-embed]`. Walks `pages WHERE type = 'code'` in 100-row batches, replays through `importCodeFile` for chunk + embed + content_hash folding. Idempotent unless `--force` bypasses the content_hash early-return. **v0.37.3.0:** cost-preview model field now reads `getEmbeddingModelName()` from the gateway instead of the back-compat `EMBEDDING_MODEL` constant — preview reflects what the gateway will actually embed with. Same wave adds an informational stderr nudge inside `runReindexCode` (not the CLI wrapper, so dry-run + execute both surface it): when the configured embedding model isn't code-tuned (allowlist currently `{'voyage-code-3'}`, case-insensitive bare match against gateway-returned name), prints a 4-line recommendation to switch to `voyage:voyage-code-3`. Suppress with `GBRAIN_NO_CODE_MODEL_NUDGE=1`, `--no-embed`, or `--json`. Pure `shouldNudgeCodeModel(bareName)` helper returns a tagged `NudgeDecision` union; the helper takes the bare model name (matches gateway return shape) and emits qualified `voyage:voyage-code-3` for the paste-ready `gbrain config set` line. Pinned by `test/ai/voyage-code-3-recipe.test.ts`, `test/reindex-code-nudge.serial.test.ts`, and `test/reindex-code-model-source.serial.test.ts` (the latter is the IRON-RULE regression for the cost-preview fix). - `src/commands/sync.ts:resolveSlugByPathOrSourcePath` (v0.32.7 CJK wave, codex post-merge F4) — Resolves a slug by `pages.source_path` first (returns the stored slug for frontmatter-fallback pages whose path doesn't derive a slug), then falls back to `resolveSlugForPath(path)`. Threaded into all 4 delete/rename call sites (`performSync`'s un-syncable cleanup at ~:531, deletes at ~:603, rename oldSlug at ~:622). Without this, emoji-only / Thai / Arabic filenames whose slug came from frontmatter would orphan on delete/rename (the delete path would compute the wrong path-derived slug). Best-effort query — pre-migration brains fall through to the legacy path. - `src/core/utils.ts` — Shared SQL utilities extracted from postgres-engine.ts. Exports `parseEmbedding(value)` (throws on unknown input, used by migration + ingest paths where data integrity matters) and as of v0.12.3 `tryParseEmbedding(value)` (returns `null` + warns once per process, used by search/rescore paths where availability matters more than strictness). **v0.26.9 (D14):** adds `isUndefinedColumnError(err)` predicate — pattern-matches Postgres SQLSTATE 42703 / "column ... does not exist" with engine-driver shape variation tolerated. Replaces bare `catch {}` blocks in `oauth-provider.ts` so genuine errors (lock timeout, network blip, permission denied) propagate while column-missing falls through to the legacy fallback path. Reusable from any future code that needs the same column-existence probe semantics. **v0.32.8 (PR #860):** adds `validateSourceId(id)` that throws on anything outside `^[a-z0-9_-]+$`. Used by the per-source disk-layout fix in patterns.ts/synthesize.ts before any `join(brainDir, '.sources', source_id, slug+'.md')` call so source_id can't traverse out of brainDir. `rowToPage` updated to populate the now-required `Page.source_id` field from the SELECT projection (`scripts/check-source-id-projection.sh` enforces that every projection feeding `rowToPage` includes the column). - `src/core/db.ts` — Connection management, schema initialization. v0.22.1 (#363, contributed by @orendi84): `resolveSessionTimeouts()` returns `statement_timeout` + `idle_in_transaction_session_timeout` (defaults: 5min each, env-overridable via `GBRAIN_STATEMENT_TIMEOUT` / `GBRAIN_IDLE_TX_TIMEOUT` / `GBRAIN_CLIENT_CHECK_INTERVAL`). Both `connect()` (module singleton) and `PostgresEngine.connect()` (worker pool) consume the result via postgres.js's `connection` option, sending GUCs as startup parameters that survive PgBouncer transaction mode (unlike the prior `setSessionDefaults` post-pool SET, kept as a back-compat no-op shim). - `src/commands/migrate-engine.ts` — Bidirectional engine migration (`gbrain migrate --to supabase/pglite`) -- `src/core/import-file.ts` — importFromFile + importFromContent (chunk + embed + tags). **v0.41.31.0:** `importFromContent` and `importCodeFile` stamp `pages.embedding_signature` via `setPageEmbeddingSignature(slug, {sourceId, signature: currentEmbeddingSignature()})` when the import actually embedded (not `--no-embed`) — covers the inline import/sync path so a model/dims swap is detectable as stale. `importCodeFile` only stamps when every chunk was freshly embedded this call (`needsEmbedIndexes.length === chunks.length`); mixed reuse-by-hash pages stay unstamped (`reindex --code --force` / `embed --stale` handle the swap for those). +- `src/core/import-file.ts` — importFromFile + importFromContent (chunk + embed + tags). **v0.41.31.0:** `importFromContent` and `importCodeFile` stamp `pages.embedding_signature` via `setPageEmbeddingSignature(slug, {sourceId, signature: currentEmbeddingSignature()})` when the import actually embedded (not `--no-embed`) — covers the inline import/sync path so a model/dims swap is detectable as stale. `importCodeFile` only stamps when every chunk was freshly embedded this call (`needsEmbedIndexes.length === chunks.length`); mixed reuse-by-hash pages stay unstamped (`reindex --code --force` / `embed --stale` handle the swap for those). **v0.41.37.0 (#1621):** `importFromContent`'s tag reconciliation is now ADD-ONLY. The pre-fix "remove every existing tag not in current frontmatter, then add current" logic wiped ALL DB-side enrichment tags (auto-tag / dream synthesize / signal-detector) on every re-import — most visibly under `gbrain reindex --markdown`, which re-imports every page with `forceRechunk`. The `tags` table has no provenance column and frontmatter tags are stripped from stored `pages.frontmatter` (markdown.ts:118), so at re-import time a frontmatter-origin tag can't be distinguished from a DB-enrichment tag — deletion is unsafe. Now it only `addTag` (idempotent, ON CONFLICT DO NOTHING). Accepted trade-off: removing a tag from frontmatter no longer removes it from the DB on next sync (needs a `tag_source` provenance column, deferred — TODOS.md #1621-followup). Pinned by `test/reindex-preserve-tags.test.ts` + `test/import-file.test.ts`. - `src/core/sync.ts` — Pure sync functions (manifest parsing, filtering, slug conversion). **v0.35.5.0:** new exported `pruneDir(name: string): boolean` helper is the single source of truth for descent-time directory exclusion across walkers. Blocks `node_modules` (no leading dot, so pre-v0.35.5 walkers slipped through and inflated MISSING_OPEN counts via vendor packages), dot-prefix dirs, `ops/`, and `*.raw` sidecars. `isSyncable` now applies it per path segment; `walkMarkdownFiles` in `src/commands/extract.ts` and `listTextFiles` in `src/core/cycle/transcript-discovery.ts` consult it BEFORE recursing so the IO cost of walking thousands of vendor files is saved. Closes #923 + #202. `manageGitignore` worktree fix in same wave: discriminator now matches the gitdir path segment (`/modules/` = submodule, `/worktrees/` = worktree, per Git's documented layout) instead of the legacy absolute-vs-relative check that misclassified absorbed submodules and worktrees both. Conductor worktrees are first-class repos and now get `.gitignore` management for storage-tiering. Closes #889. v0.22.12 (#500, foundation by @wintermute via #501): `classifyErrorCode(errorMsg)` regex-based classifier with 12 codes (`SLUG_MISMATCH`, `YAML_PARSE`, `YAML_DUPLICATE_KEY`, `MISSING_OPEN`, `MISSING_CLOSE`, `NESTED_QUOTES`, `EMPTY_FRONTMATTER`, `NULL_BYTES`, `INVALID_UTF8`, `STATEMENT_TIMEOUT`, `FILE_TOO_LARGE`, `SYMLINK_NOT_ALLOWED`) plus `UNKNOWN` fallback. `summarizeFailuresByCode(failures)` returns sorted `[{code, count}]`. `code?` optional field on `SyncFailure`; backfilled at ack time on pre-v0.22.12 entries. `acknowledgeSyncFailures()` returns `AcknowledgeResult { count, summary }`. Three regexes (`MISSING_OPEN`, `MISSING_CLOSE`, `EMPTY_FRONTMATTER`) broadened to match actual `markdown.ts:159-244` validator message strings, not just the literal code-name prefix. `FILE_TOO_LARGE` covers all three production size sites in `import-file.ts:199, 352, 401`; `SYMLINK_NOT_ALLOWED` covers the rejection at `:347`. Closes the silent-skip pattern that motivated #500. - `src/core/storage.ts` — Pluggable storage interface (S3, Supabase Storage, local) - `src/core/storage-config.ts` (v0.22.11) — Storage tiering: `loadStorageConfig` reads `gbrain.yml`, normalizes deprecated keys (`git_tracked` / `supabase_only`) to canonical (`db_tracked` / `db_only`) with once-per-process deprecation warning, and runs `normalizeAndValidateStorageConfig` (auto-fixes missing trailing `/`, throws `StorageConfigError` on tier overlap). Path-segment matcher: `media/x/` does NOT match `media/xerox/foo`. Replaces gray-matter (broken on delimiter-less YAML) with a dedicated parser for the `gbrain.yml` shape. @@ -402,7 +402,7 @@ strict behavior when unset. - `admin/` (v0.26.0) — React 19 + Vite + TypeScript admin SPA embedded in the binary via `admin/dist/` served by `serve-http.ts`. 7 screens: Login (bootstrap token → session cookie), Dashboard (metrics + SSE feed + token health), Agents (sortable table + sparklines + Register button), Register (modal with scope checkboxes + grant type selector), Credentials reveal (full-screen modal with Copy + Download JSON + yellow one-time-only warning), Request Log (filterable paginated), Agent Detail drawer (Details / Activity / Config Export tabs + Revoke). Design tokens: `#0a0a0f` bg, Inter for UI, JetBrains Mono for data, 4-32px spacing scale, rounded pill badges. HTTP-only SameSite=Strict cookie auth. 65KB gzip. Build: `cd admin && bun install && bun run build`; output at `admin/dist/` is committed for self-contained binaries. - `src/commands/auth.ts` — Token management. `gbrain auth create/list/revoke/test` for legacy bearer tokens (v0.22.7 wired as a first-class CLI subcommand) plus `gbrain auth register-client` (v0.26.0) and `gbrain auth revoke-client ` (v0.26.2) for OAuth 2.1 client lifecycle. `revoke-client` runs an atomic `DELETE...RETURNING` on `oauth_clients`; FK `ON DELETE CASCADE` on `oauth_tokens.client_id` and `oauth_codes.client_id` purges every active token + authorization code in a single transaction. `process.exit(1)` on no-such-client (idempotent — re-running on the same id produces the same exit-1 message). Legacy tokens stored as SHA-256 hashes in `access_tokens`; OAuth clients in `oauth_clients`. As of v0.26.0, legacy tokens grandfather to `read+write+admin` scopes on the OAuth HTTP server, so pre-v0.26 deployments keep working with no migration. **v0.31.3 (#681):** every SQL site routes through `sqlQueryForEngine(engine)` from `src/core/sql-query.ts` (and `executeRawJsonb` for the takes-holders `permissions` JSONB column) so `gbrain auth` works against PGLite brains. Pre-fix, every call hit the postgres.js singleton via `getConn()` and silently failed (or wrote to the wrong DB) when the active engine was PGLite. The takes-holders write goes through `executeRawJsonb(engine, sql, [name, hash], [{takes_holders:[...]}])` which round-trips with `jsonb_typeof = 'object'` instead of the pre-v0.31.3 quoted-string shape. **v0.34.1.0 (#876):** `register-client` accepts `--source ` (write authority, scalar) and `--federated-read ` (read scope, array). The output prints the resolved `Write source` and `Federated reads` for the registered client. Pre-v0.34 clients backfill to `source_id='default'` via migration v60 so existing deployments keep their v0.33 effective behavior verbatim. - `src/commands/upgrade.ts` — Self-update CLI. `runPostUpgrade()` enumerates migrations from the TS registry (src/commands/migrations/index.ts) and tail-calls `runApplyMigrations(['--yes', '--non-interactive'])` so the mechanical side of every outstanding migration runs unconditionally. -- `src/commands/migrations/` — TS migration registry (compiled into the binary; no filesystem walk of `skills/migrations/*.md` needed at runtime). `index.ts` lists migrations in semver order. `v0_11_0.ts` = Minions adoption orchestrator (8 phases). `v0_12_0.ts` = Knowledge Graph auto-wire orchestrator (5 phases: schema → config check → backfill links → backfill timeline → verify). `phaseASchema` has a 600s timeout (bumped from 60s in v0.12.1 for duplicate-heavy brains). `v0_12_2.ts` = JSONB double-encode repair orchestrator (4 phases: schema → repair-jsonb → verify → record). `v0_14_0.ts` = shell-jobs + autopilot cooperative (2 phases: schema ALTER minion_jobs.max_stalled SET DEFAULT 3 — superseded by v0.14.3's schema-level DEFAULT 5 + UPDATE backfill; pending-host-work ping for skills/migrations/v0.14.0.md). All orchestrators are idempotent and resumable from `partial` status. As of v0.14.2 (Bug 3), the RUNNER owns all ledger writes — orchestrators return `OrchestratorResult` and `apply-migrations.ts` persists a canonical `{version, status, phases}` shape after return. Orchestrators no longer call `appendCompletedMigration` directly. `statusForVersion` prefers `complete` over `partial` (never regresses). 3 consecutive partials → wedged → `--force-retry ` writes a `'retry'` reset marker. v0.14.3 (fix wave) ships schema-only migrations v14 (`pages_updated_at_index`) + v15 (`minion_jobs_max_stalled_default_5` with UPDATE backfill) via the `MIGRATIONS` array in `src/core/migrate.ts` — no orchestrator phases needed. +- `src/commands/migrations/` — TS migration registry (compiled into the binary; no filesystem walk of `skills/migrations/*.md` needed at runtime). `index.ts` lists migrations in semver order. `v0_11_0.ts` = Minions adoption orchestrator (8 phases). `v0_12_0.ts` = Knowledge Graph auto-wire orchestrator (5 phases: schema → config check → backfill links → backfill timeline → verify). `phaseASchema` has a 600s timeout (bumped from 60s in v0.12.1 for duplicate-heavy brains). `v0_12_2.ts` = JSONB double-encode repair orchestrator (4 phases: schema → repair-jsonb → verify → record). `v0_14_0.ts` = shell-jobs + autopilot cooperative (2 phases: schema ALTER minion_jobs.max_stalled SET DEFAULT 3 — superseded by v0.14.3's schema-level DEFAULT 5 + UPDATE backfill; pending-host-work ping for skills/migrations/v0.14.0.md). All orchestrators are idempotent and resumable from `partial` status. As of v0.14.2 (Bug 3), the RUNNER owns all ledger writes — orchestrators return `OrchestratorResult` and `apply-migrations.ts` persists a canonical `{version, status, phases}` shape after return. Orchestrators no longer call `appendCompletedMigration` directly. `statusForVersion` prefers `complete` over `partial` (never regresses). 3 consecutive partials → wedged → `--force-retry ` writes a `'retry'` reset marker. v0.14.3 (fix wave) ships schema-only migrations v14 (`pages_updated_at_index`) + v15 (`minion_jobs_max_stalled_default_5` with UPDATE backfill) via the `MIGRATIONS` array in `src/core/migrate.ts` — no orchestrator phases needed. **v0.41.37.0 (#1605):** new `in-process.ts` exports `runMigrateOnlyCore({timeoutMs?})` — the single source of truth for "bring schema to head" (`configureGateway` → `createEngine` → `connect` → `initSchema` → `disconnect`, idempotent, 600s wall-clock guard `MIGRATE_ONLY_TIMEOUT_MS`, throws `MigrateOnlyError` on no-config / timeout). The migration orchestrators' 9 schema phases AND `init.ts:initMigrateOnly` (the `gbrain init --migrate-only` CLI path) both delegate to it, so the schema bring-up can't drift between them. Previously each phase shelled out to a child `gbrain init --migrate-only` via `execSync`; on Windows + bun + Supabase pooler the SPAWNED child died with `getaddrinfo ENOTFOUND` before it could connect (a bun-on-Windows child-process DNS failure, NOT an env-propagation bug — the parent connects fine with `env: process.env`). Running in-process removes the spawn entirely. `runGbrainSubprocess` is the diagnostic wrapper for the REMAINING non-schema spawns (extract/repair/stats): captures child stderr (64MB buffer) and folds it into the thrown error so a Windows failure shows the real `getaddrinfo ENOTFOUND` line instead of a bare `Command failed`. Pinned by `test/migration-in-process.serial.test.ts`. **v0.41.37.0 (#1581):** `v0_13_1.ts:phaseCGrandfather` rewritten from a per-page `getPage` + `putPage` loop (which hung CPU-bound 70+ min on an 82K-page PGLite brain) to a CHUNKED bulk SQL pass. Three correctness properties the per-page loop and a naive single bulk UPDATE both got wrong: (1) keyed on `pages.id` (globally unique PK), NOT slug — slug uniqueness is `(source_id, slug)`, so a slug-batched UPDATE would mutate same-slug pages across other sources; (2) filters `deleted_at IS NULL` so tombstones aren't grandfathered; (3) chunked in `CHUNK_SIZE` batches (DELETE_BATCH_SIZE convention) so lock-hold stays bounded instead of one giant transaction. The rollback log carries source identity (`{id, slug, source_id, pre_frontmatter}`) so a rollback is unambiguous across sources. Idempotent + resumable (each UPDATE flips its rows out of `GRANDFATHER_WHERE`). Completes ~1-2s on the 82K-page brain. Pinned by `test/migrations-v0_13_1-grandfather.test.ts`. - `src/commands/repair-jsonb.ts` — `gbrain repair-jsonb [--dry-run] [--json]`: rewrites `jsonb_typeof='string'` rows in place across 5 affected columns (pages.frontmatter, raw_data.data, ingest_log.pages_updated, files.metadata, page_versions.frontmatter). Fixes v0.12.0 double-encode bug on Postgres; PGLite no-ops. Idempotent. - `src/commands/orphans.ts` — `gbrain orphans [--json] [--count] [--include-pseudo] [--source ]`: surfaces pages with zero inbound wikilinks, grouped by domain. Auto-generated/raw/pseudo pages filtered by default. Also exposed as `find_orphans` MCP operation. Shipped in v0.12.3 (contributed by @knee5). **v0.41.29.0 source-scoping wave:** `findOrphans`/`getOrphansData` (the canonical pure fn shared with doctor's `orphan_ratio`) takes `{ sourceId?, sourceIds? }`; `BrainEngine.findOrphanPages(opts?)` (both engines) scopes ONLY the candidate side (`p.source_id = $1` scalar, or `= ANY($1::text[])` federated) while still counting inbound links from ANY source — a page in source X linked FROM source Y is reachable, so NOT an orphan of X (the deliberate, less-surprising definition; the stricter intra-source-only reading is rejected). `--source` is an explicit raw-flag parse (NOT `resolveSourceWithTier`, which would scope bare invocations to a default). Also corrected the `total_linkable` denominator: it now enumerates ALL live pages (scoped) and subtracts every excluded-by-slug page (templates/, scratch/, etc.) — pre-fix `total - excludedOrphans` left excluded NON-orphan pages with inbound links in the denominator, inflating it and suppressing warnings (changes orphan_ratio output for every brain, in the accurate direction). The `find_orphans` MCP op threads `sourceScopeOpts(ctx)` so a source-bound OAuth client no longer sees brain-wide orphans (closes a cross-source read leak in the v0.34.1 class). `gbrain doctor --source ` scopes `orphan_ratio` (entity-count gate + getOrphansData + messages) and, under explicit `--source` below 100 entity pages, reports the ratio with a low-scale caveat instead of a vacuous "ok". Thin-client `doctor --source` orphan_ratio remains brain-wide (TODO). Pinned by `test/orphans-source-scope.test.ts` (PGLite) + `test/e2e/engine-parity.test.ts` (Postgres↔PGLite scalar + federated parity). - `src/commands/salience.ts` (v0.29) — `gbrain salience [--days N] [--limit N] [--kind PREFIX] [--json]`: pages ranked by emotional + activity salience over a recency window. Mirrors orphans.ts shape (pure data fn + JSON formatter + human formatter). Calls `engine.getRecentSalience(opts)`. Score formula: `(emotional_weight × 5) + ln(1 + active_take_count) + 1/(1 + days_since_update)`. @@ -429,7 +429,7 @@ strict behavior when unset. - `src/core/embedding-dim-check.ts` extension (v0.41.17.0, T5+T6) — facts.embedding dim drift surface. New `readFactsEmbeddingDim(engine): Promise` covers both `vector(N)` and `halfvec(N)` shapes (codex #19 — migration v40 falls back to `vector` on pgvector < 0.7). Regex ordering halfvec-before-vector pinned by tests (substring "vec" appears in "halfvec"; naive `/vector/i` would shadow). New `buildFactsAlterRecipe(dims, configured, type)` emits the paste-ready `DROP INDEX IF EXISTS idx_facts_embedding_hnsw; ALTER TABLE facts ALTER COLUMN embedding TYPE halfvec(N) USING embedding::halfvec(N); CREATE INDEX idx_facts_embedding_hnsw ON facts USING hnsw (embedding halfvec_cosine_ops) WHERE ...` flow (codex #18 — NOT bare REINDEX which doesn't rewrite the index after a column-type change). New `assertFactsEmbeddingDimMatchesConfig(engine)` is the D15 preflight — throws `FactsEmbeddingDimMismatchError` (tagged class with `tag: 'FACTS_EMBEDDING_DIM_MISMATCH'` for parity with the worker-pool's MUST_ABORT semantics) when configured dim doesn't match the column width. Result cached per-engine via `WeakMap`. PGLite engines silently skip. New doctor check `facts_embedding_width_consistency` (registered in `runDoctor` after `embedding_width_consistency`) reuses the same helpers — surfaces drift with paste-ready ALTER recipe identical to the preflight error. Pinned by 18 cases in `test/embedding-dim-check-facts.test.ts`. - `src/core/postgres-engine.ts` extension (v0.41.17.0, T6, codex #20) — `insertFact` + `insertFacts` no longer hardcode `tx.unsafe(\`'${embedLit}'::vector\`)` for the embedding column. New `resolveFactsEmbeddingCast()` private method probes `pg_attribute` once per engine instance (cached in `_factsEmbeddingCastSuffix`) and returns `'::halfvec'` when migration v40 created the column as halfvec, else `'::vector'`. Both insert paths use the cached suffix so the cast matches the actual column type. Pre-fix all three insert sites hardcoded `::vector`; works on pgvector >= 0.7 via implicit auto-cast but fails on older pgvector. Test seam `__resetFactsEmbeddingCastCacheForTest()` clears per-engine cache. - `src/core/cycle.ts` + `src/core/cycle/extract-atoms.ts` + `src/core/cycle/synthesize-concepts.ts` + `src/commands/extract.ts` + `src/commands/doctor.ts` + `src/core/op-checkpoint.ts` extensions (v0.41.20.0 ops-fix-wave) — six daily-driver ops pains in one bisectable wave. (1) **Batch idempotency for `extract_atoms`:** new `atomsExistingForHashes(engine, sourceId, hashes[])` exported from `src/core/cycle/extract-atoms.ts` replaces the per-hash loop that did 7K individual queries at the start of every cycle on brains with conversation-transcript corpora (5-10 min silent overhead). One batched SQL roundtrip returns the set of `content_hash16` values already extracted as atoms for this source. Fail-open: SQL error logs to stderr and returns an empty set so extraction proceeds. Powered by migration v104 `pages_atom_source_hash_idx` (partial expression index on `frontmatter->>'source_hash'` for atom rows where `deleted_at IS NULL`; Postgres uses `CREATE INDEX CONCURRENTLY` with invalid-remnant pre-drop mirroring v97 `pages_dedup_partial_index`, PGLite uses plain `CREATE INDEX`). Without this index the batch idempotency check would seq-scan pages on big brains and defeat the perf win. (2) **Shorter cycle lock TTL + active in-phase refresh:** `LOCK_TTL_MINUTES` dropped from 30 → 5 in `src/core/cycle.ts`. New exported `buildYieldDuringPhase(lock, outer)` closure calls `lock.refresh()` AND any external hook on every fire; throttled to 30s inside each phase via `maybeYield`. Fires both inside the main work loop AND immediately after every `await chat(...)` LLM call so long Haiku/Sonnet calls don't sit past TTL. `LockHandle` and `buildYieldDuringPhase` exported for test seam access. `synthesize_concepts` no longer fires `yieldDuringPhase` per-concept-group — same hook, throttled via the new shared `maybeYield` helper. A crashed cycle releases its lock 6x faster while a healthy long-running cycle keeps the lock alive. Known residual under shorter TTL: a single `await chat()` past 5 min wallclock can expire the lock mid-await without the original phase noticing — `DbLockHandle.refresh()` throwing on 0 rows affected is filed as P2 follow-up TODO-OPS-2. (3) **Progress wiring through long phases:** new `progress?: ProgressReporter` opt on `ExtractAtomsOpts` and `SynthesizeConceptsOpts`. Cycle.ts passes its phase-level reporter down (NOT a child reporter — that would produce a path collision `cycle.extract_atoms.extract_atoms.work`). Phases only call `tick()` and `heartbeat()`; cycle.ts owns `start()` and `finish()`. You see `[cycle.extract_atoms] N (atoms_created)` ticks every ~1s during both long phases instead of "start" then silence for 10+ min. (4) **`by-mention` resume:** new `mentionsFingerprint({source, type, since, gazetteerHash})` in `src/core/op-checkpoint.ts`. The gazetteer hash is the load-bearing field — adding new entity pages mid-pause shifts the hash, gets a new fingerprint, and triggers a fresh scan against the new gazetteer instead of silently skipping previously-scanned pages. `gbrain extract links --by-mention` now resumes from where it died via the existing `op_checkpoints` framework with a `flushAndCheckpoint` ordering — links flush to the DB FIRST, page keys commit to the checkpoint SECOND, persist THIRD. A crash between `batch.push()` and flush leaves the page un-checkpointed so resume re-scans it. Persist cadence: every 1000 items OR every 30s, whichever first. Clean exit clears the checkpoint. `--dry-run` deliberately skips both load and write so it stays an inspection mode. (5) **`sync_consolidation` doctor check:** multi-source brains see a paste-ready `gbrain sync --all --parallel 4 --workers 4 --skip-failed` recommendation. Single-source brains get "not applicable." SQL errors return `warn` via the check's own try/catch — outer doctor catch isn't a safe assumption. Companion "Multi-source brains" recipe block in `skills/cron-scheduler/SKILL.md` documents the `sync --all` pattern as preferred over per-source entries. (6) **Test-isolation fixes:** `test/cycle-last-full-cycle-at.test.ts` + `test/schema-cli.test.ts` migrated to per-test `GBRAIN_HOME=tempdir` isolation. Pinned by 44 new unit/PGLite cases across 9 files: `test/cycle/extract-atoms-batch.test.ts` (5 — batch idempotency), `test/cycle/cycle-lock-ttl.test.ts` (1 — regression pin on `LOCK_TTL_MINUTES === 5`), `test/op-checkpoint-mentions-fingerprint.test.ts` (7 — fingerprint sensitivity including gazetteer-hash regression guard), `test/cycle/extract-atoms-progress.test.ts` (4 — phase doesn't call start/finish, ticks fire per item), `test/cycle/synthesize-concepts-progress.test.ts` (3 — same shape), `test/cycle/yield-during-phase-refresh.test.ts` (7 — buildYieldDuringPhase actually calls lock.refresh() + outer hook, throws non-fatal), `test/cycle/yield-during-phase-throttle.test.ts` (3 — 30s throttle gate), `test/extract-by-mention-resume.test.ts` (5 — checkpoint persistence ordering, dry-run skips persist, gazetteer change invalidates, filtered pages get checkpointed), `test/doctor-sync-consolidation.test.ts` (6 — edge case matrix for source counts + archived filtering + SQL error path). Two follow-up TODOs filed in TODOS.md under "v0.41.19.0 ops-fix-wave follow-ups": `gbrain sync print-cron` subcommand (TODO-OPS-1) and lock-loss detection via `DbLockHandle.refresh()` throwing on 0 rows affected (TODO-OPS-2). -- `src/commands/sync.ts` — `gbrain sync` CLI + the `performSync` / `performFullSync` library entrypoints (consumed by the autopilot cycle and the Minion sync handler). v0.22.13 (PR #490): `performSync` wraps its body in a `gbrain-sync` writer lock so two concurrent syncs (manual + autopilot, two terminals, two Conductor workspaces) cannot both write `last_commit` and let the last writer win. Head-drift gate after the import phase re-checks `git rev-parse HEAD`; if HEAD moved (someone ran `git checkout` / `git pull` mid-sync), the bookmark refuses to advance. Vanished files now record a failedFiles entry instead of silent-skip — the silent-skip-then-advance pathology that survived prior hardening passes is dead. Worker engines wrap in try/finally so disconnect always fires (panic-path leak fix). Both PGLite-detection sites use `engine.kind === 'pglite'`. CLI accepts `--workers N` (alias `--concurrency N`), validated via `parseWorkers`. Explicit `--workers` bypasses the auto-path file-count floor; auto path defers to `autoConcurrency()`. Banner moved to stderr. **v0.34.2.0:** the inline `.sort()` over add/mod paths is replaced with `sortNewestFirst(addsAndMods)` from `src/core/sort-newest-first.ts`, so the newest-first descending-lex policy lives in one helper shared with `gbrain import` instead of drifting across two files. **v0.40.3.0 (productionized from PR #1314 by @garrytan-agents):** the load-bearing change is the lock-identity invariant — `performSync` now defaults to a per-source lock id (`gbrain-sync:`) whenever `opts.sourceId` is set, AND wraps the writer window in `withRefreshingLock` from `src/core/db-lock.ts` so long-running sources (the PR motivator: media-corpus / 250K+ chunks) don't lose their lock at the 30-min TTL mid-run. Legacy single-default-source brains keep the bare `tryAcquireDbLock(SYNC_LOCK_ID)` path for back-compat. `SyncOpts.lockId?: string` is the explicit override (escape hatch; production callers don't set it). The fix closes the bug class Codex's outside-voice review caught in the original PR — a `sync --all` worker on per-source lock racing against `sync --source foo` on global lock would have silently corrupted the same source. `gbrain sync --all` got a continuous worker pool: `parseWorkers`-validated `--parallel N` (default `min(sourceCount, --workers, DEFAULT_PARALLEL_SOURCES=4)` from `src/core/sync-concurrency.ts`), `parallel` long-lived async workers pulling from a shared FIFO queue (no head-of-line blocking from wave dispatch), per-source `withSourcePrefix(src.id, ...)` wrap from `src/core/console-prefix.ts` so every `slog`/`serr` line carries `[] ` for kubectl-style greppable parallel output. `--skip-failed` and `--retry-failed` reject with a paste-ready error when combined with `--parallel > 1` (the brain-global `sync-failures.jsonl` has no per-source scope; v0.41+ TODO source-scopes it). Connection-budget stderr warning fires when `parallel × workers × 2 > 16` (the `× 2 per-file pool` factor: each per-file worker opens its own `PostgresEngine` with `poolSize=2`, so `--parallel 4 --workers 4` is actually 32 connections, not 16 — Codex caught the 2× understatement in the original plan). New exports `resolveParallelism`, `syncOneSource`, `buildSyncStatusReport`, `printSyncStatusReport`, `SyncStatusReport` back the `gbrain sources status` dashboard surface (D3 → `sources` subcommand, not a `sync` flag, so reads and writes don't share a verb). Stable `--json` envelope `{schema_version: 1, sources, parallel, ok_count, error_count}` on stdout under `--json`; human banners route to stderr via the `humanSink` helper so `jq` parses cleanly. Exit matrix: 0 = all ok, 1 = any error, 2 = cost-prompt-not-confirmed (unchanged). The dashboard SQL is the canonical `content_chunks ch JOIN pages pg ON pg.id = ch.page_id WHERE pg.deleted_at IS NULL` shape with `archived = false` filter at the caller — the original PR shipped `chunks ch JOIN ON page_slug`, which would have crashed on PGLite parse and silently zeroed on Postgres via the swallow-catch. Embedding column resolved via `resolveEmbeddingColumn(undefined, cfg)` from `src/core/search/embedding-column.ts` so Voyage / multimodal / non-default-column brains see counts against the column they actually use. Errors propagate from the dashboard SQL — no swallow-catch (Q2 sub-fix). The IRON RULE regression lives in `test/e2e/sync-status-pglite.test.ts`: real PGLite seeds 2 sources × pages × chunks, soft-deletes 1 page, archives 1 source, validates the SQL excludes both AND the active embedding column is the one used. 38 in-file `console.log`/`console.error` call sites inside `performSync*` migrated to `slog`/`serr` (top-level `runSync` orchestrator console calls intentionally stay outside the prefix scope). **v0.41.25.0 (supersedes PR #1538):** sync delete loop rewritten as interleaved per-batch resolve+delete using `engine.resolveSlugsByPaths` + `engine.deletePages` from `src/core/engine.ts`. 73K-delete commit drops from ~146K SQL round-trips (~5h on Postgres+pgbouncer) to ~292 round-trips (~2min) — closes the cascade-staleness bug class where one big-delete commit jammed every other source's sync for hours. Per-batch try-catch decomposes batch DELETE failures to per-slug `deletePage` fallback so a transient blip on batch 73 doesn't lose 500 deletes; unrecoverable per-slug failures land in `failedFiles` (matches the existing import-loop pattern). `pagesAffected` filters to D6 confirmed-deleted slugs (downstream extract/embed stop wasting work on phantoms). Same batched slug-resolve treatment for the rename loop (`gbrain sync` renames — Phase 3 in the plan; the per-file `importFile` stays per-file, only the slug-resolve N+1 gets batched). The `resolveSlugByPathOrSourcePath` single-call helper at `sync.ts:267` now delegates to `engine.resolveSlugsByPaths` when `sourceId` is set (D8 DRY — one owner of the SQL + fallback semantics), keeping legacy `executeRaw` fallback for the no-sourceId path which is functionally dead post-v0.34.1's source-resolution wiring. Legacy no-sourceId branch in the delete loop preserves correctness on a slow path; the engine batch methods require sourceId so the new fast path only fires when production threading is intact. `failedFiles` declaration hoisted to top of `performSyncInner` so both delete decompose and import loops feed the same sync-bookmark gate. **v0.41.31.0 (mode-aware cost gate):** the v0.20.0 unconditional `sync --all` ConfirmationRequired gate is superseded by a mode-aware gate driven by `willEmbedSynchronously` + `shouldBlockSync` from `src/core/embedding.ts`. federated_v2 is resolved ONCE up front (shared with the fan-out below). On the DEFERRED path (v2 on, parallel) embedding goes to per-source `embed-backfill` jobs that carry their own `$X/source/24h` cap (default $25, `SPEND_CAP_CONFIG_KEY` from `embed-backfill-submit.ts`), so the gate is INFORMATIONAL — it prints an FYI naming the backfill cap + current backlog (`sumStaleChunkChars({signature})` priced via `estimateCostFromChars` at `currentEmbeddingPricePerMTok()`) + the count of already-queued `embed-backfill` jobs (TODO-2 visibility), and NEVER exits 2. On the INLINE path (v2 off, or `--serial` without `--no-embed`) the BLOCKING gate fires only when the NEW-CONTENT estimate exceeds the `sync.cost_gate_min_usd` floor (default $0.50). The new-content estimate (`estimateInlineNewTokens`) contributes ZERO for sources provably unchanged since last sync (`isSourceUnchangedSinceSync` HEAD==last_commit + clean tree, AND `chunker_version === CURRENT`) — mirrors doctor's `sync_freshness` + sync's own do-work gate — and the full-tree ceiling for changed sources. F2 fix: the pre-existing stale backlog (NULL embeddings + signature drift) is shown informationally but NEVER gated on, because sync doesn't sweep it inline (`gbrain embed --stale` does); gating on it would block every inline cron after a model swap. New helpers `resolveCostGateFloorUsd(engine)` (reads `sync.cost_gate_min_usd`, fail-open $0.50, accepts 0) + `resolveBackfillCapUsd(engine)`. The `sync --all` SELECT widens to carry `last_commit + chunker_version` (both columns predate v0.41; no migration). JSON envelopes gain `mode` + `gate` discriminators (`dry_run | deferred_notice | below_floor | confirmation_required`). `SyncStatusReportSource` gains `backfill_queued` / `backfill_active` / `backfill_last_completed_at` (best-effort; 0/null on pre-minions brains). All embedding-cost preview math reads the configured model name via `getEmbeddingModelName()` (no hardcoded OpenAI). Pinned by `test/sync-cost-gate.serial.test.ts` + `test/sync-cost-preview.test.ts`. +- `src/commands/sync.ts` — `gbrain sync` CLI + the `performSync` / `performFullSync` library entrypoints (consumed by the autopilot cycle and the Minion sync handler). v0.22.13 (PR #490): `performSync` wraps its body in a `gbrain-sync` writer lock so two concurrent syncs (manual + autopilot, two terminals, two Conductor workspaces) cannot both write `last_commit` and let the last writer win. Head-drift gate after the import phase re-checks `git rev-parse HEAD`; if HEAD moved (someone ran `git checkout` / `git pull` mid-sync), the bookmark refuses to advance. Vanished files now record a failedFiles entry instead of silent-skip — the silent-skip-then-advance pathology that survived prior hardening passes is dead. Worker engines wrap in try/finally so disconnect always fires (panic-path leak fix). Both PGLite-detection sites use `engine.kind === 'pglite'`. CLI accepts `--workers N` (alias `--concurrency N`), validated via `parseWorkers`. Explicit `--workers` bypasses the auto-path file-count floor; auto path defers to `autoConcurrency()`. Banner moved to stderr. **v0.34.2.0:** the inline `.sort()` over add/mod paths is replaced with `sortNewestFirst(addsAndMods)` from `src/core/sort-newest-first.ts`, so the newest-first descending-lex policy lives in one helper shared with `gbrain import` instead of drifting across two files. **v0.40.3.0 (productionized from PR #1314 by @garrytan-agents):** the load-bearing change is the lock-identity invariant — `performSync` now defaults to a per-source lock id (`gbrain-sync:`) whenever `opts.sourceId` is set, AND wraps the writer window in `withRefreshingLock` from `src/core/db-lock.ts` so long-running sources (the PR motivator: media-corpus / 250K+ chunks) don't lose their lock at the 30-min TTL mid-run. Legacy single-default-source brains keep the bare `tryAcquireDbLock(SYNC_LOCK_ID)` path for back-compat. `SyncOpts.lockId?: string` is the explicit override (escape hatch; production callers don't set it). The fix closes the bug class Codex's outside-voice review caught in the original PR — a `sync --all` worker on per-source lock racing against `sync --source foo` on global lock would have silently corrupted the same source. `gbrain sync --all` got a continuous worker pool: `parseWorkers`-validated `--parallel N` (default `min(sourceCount, --workers, DEFAULT_PARALLEL_SOURCES=4)` from `src/core/sync-concurrency.ts`), `parallel` long-lived async workers pulling from a shared FIFO queue (no head-of-line blocking from wave dispatch), per-source `withSourcePrefix(src.id, ...)` wrap from `src/core/console-prefix.ts` so every `slog`/`serr` line carries `[] ` for kubectl-style greppable parallel output. `--skip-failed` and `--retry-failed` reject with a paste-ready error when combined with `--parallel > 1` (the brain-global `sync-failures.jsonl` has no per-source scope; v0.41+ TODO source-scopes it). Connection-budget stderr warning fires when `parallel × workers × 2 > 16` (the `× 2 per-file pool` factor: each per-file worker opens its own `PostgresEngine` with `poolSize=2`, so `--parallel 4 --workers 4` is actually 32 connections, not 16 — Codex caught the 2× understatement in the original plan). New exports `resolveParallelism`, `syncOneSource`, `buildSyncStatusReport`, `printSyncStatusReport`, `SyncStatusReport` back the `gbrain sources status` dashboard surface (D3 → `sources` subcommand, not a `sync` flag, so reads and writes don't share a verb). Stable `--json` envelope `{schema_version: 1, sources, parallel, ok_count, error_count}` on stdout under `--json`; human banners route to stderr via the `humanSink` helper so `jq` parses cleanly. Exit matrix: 0 = all ok, 1 = any error, 2 = cost-prompt-not-confirmed (unchanged). The dashboard SQL is the canonical `content_chunks ch JOIN pages pg ON pg.id = ch.page_id WHERE pg.deleted_at IS NULL` shape with `archived = false` filter at the caller — the original PR shipped `chunks ch JOIN ON page_slug`, which would have crashed on PGLite parse and silently zeroed on Postgres via the swallow-catch. Embedding column resolved via `resolveEmbeddingColumn(undefined, cfg)` from `src/core/search/embedding-column.ts` so Voyage / multimodal / non-default-column brains see counts against the column they actually use. Errors propagate from the dashboard SQL — no swallow-catch (Q2 sub-fix). The IRON RULE regression lives in `test/e2e/sync-status-pglite.test.ts`: real PGLite seeds 2 sources × pages × chunks, soft-deletes 1 page, archives 1 source, validates the SQL excludes both AND the active embedding column is the one used. 38 in-file `console.log`/`console.error` call sites inside `performSync*` migrated to `slog`/`serr` (top-level `runSync` orchestrator console calls intentionally stay outside the prefix scope). **v0.41.25.0 (supersedes PR #1538):** sync delete loop rewritten as interleaved per-batch resolve+delete using `engine.resolveSlugsByPaths` + `engine.deletePages` from `src/core/engine.ts`. 73K-delete commit drops from ~146K SQL round-trips (~5h on Postgres+pgbouncer) to ~292 round-trips (~2min) — closes the cascade-staleness bug class where one big-delete commit jammed every other source's sync for hours. Per-batch try-catch decomposes batch DELETE failures to per-slug `deletePage` fallback so a transient blip on batch 73 doesn't lose 500 deletes; unrecoverable per-slug failures land in `failedFiles` (matches the existing import-loop pattern). `pagesAffected` filters to D6 confirmed-deleted slugs (downstream extract/embed stop wasting work on phantoms). Same batched slug-resolve treatment for the rename loop (`gbrain sync` renames — Phase 3 in the plan; the per-file `importFile` stays per-file, only the slug-resolve N+1 gets batched). The `resolveSlugByPathOrSourcePath` single-call helper at `sync.ts:267` now delegates to `engine.resolveSlugsByPaths` when `sourceId` is set (D8 DRY — one owner of the SQL + fallback semantics), keeping legacy `executeRaw` fallback for the no-sourceId path which is functionally dead post-v0.34.1's source-resolution wiring. Legacy no-sourceId branch in the delete loop preserves correctness on a slow path; the engine batch methods require sourceId so the new fast path only fires when production threading is intact. `failedFiles` declaration hoisted to top of `performSyncInner` so both delete decompose and import loops feed the same sync-bookmark gate. **v0.41.31.0 (mode-aware cost gate):** the v0.20.0 unconditional `sync --all` ConfirmationRequired gate is superseded by a mode-aware gate driven by `willEmbedSynchronously` + `shouldBlockSync` from `src/core/embedding.ts`. federated_v2 is resolved ONCE up front (shared with the fan-out below). On the DEFERRED path (v2 on, parallel) embedding goes to per-source `embed-backfill` jobs that carry their own `$X/source/24h` cap (default $25, `SPEND_CAP_CONFIG_KEY` from `embed-backfill-submit.ts`), so the gate is INFORMATIONAL — it prints an FYI naming the backfill cap + current backlog (`sumStaleChunkChars({signature})` priced via `estimateCostFromChars` at `currentEmbeddingPricePerMTok()`) + the count of already-queued `embed-backfill` jobs (TODO-2 visibility), and NEVER exits 2. On the INLINE path (v2 off, or `--serial` without `--no-embed`) the BLOCKING gate fires only when the NEW-CONTENT estimate exceeds the `sync.cost_gate_min_usd` floor (default $0.50). The new-content estimate (`estimateInlineNewTokens`) contributes ZERO for sources provably unchanged since last sync (`isSourceUnchangedSinceSync` HEAD==last_commit + clean tree, AND `chunker_version === CURRENT`) — mirrors doctor's `sync_freshness` + sync's own do-work gate — and the full-tree ceiling for changed sources. F2 fix: the pre-existing stale backlog (NULL embeddings + signature drift) is shown informationally but NEVER gated on, because sync doesn't sweep it inline (`gbrain embed --stale` does); gating on it would block every inline cron after a model swap. New helpers `resolveCostGateFloorUsd(engine)` (reads `sync.cost_gate_min_usd`, fail-open $0.50, accepts 0) + `resolveBackfillCapUsd(engine)`. The `sync --all` SELECT widens to carry `last_commit + chunker_version` (both columns predate v0.41; no migration). JSON envelopes gain `mode` + `gate` discriminators (`dry_run | deferred_notice | below_floor | confirmation_required`). `SyncStatusReportSource` gains `backfill_queued` / `backfill_active` / `backfill_last_completed_at` (best-effort; 0/null on pre-minions brains). All embedding-cost preview math reads the configured model name via `getEmbeddingModelName()` (no hardcoded OpenAI). Pinned by `test/sync-cost-gate.serial.test.ts` + `test/sync-cost-preview.test.ts`. **v0.41.37.0 (#1569):** new `SyncOpts.noSchemaPack` (CLI `--no-schema-pack`, threaded through `performSync` AND `syncOneSource` so `sync --all` honors it) — skips `loadActivePack` entirely so no user-supplied pack page-type regex runs during import; pages fall back to legacy prefix typing. Escape hatch for completing a sync when a suspect pack regex is the suspected cause of a wedge (re-run extraction later). Same release adds a per-file BEGIN heartbeat: `if (process.env.GBRAIN_SYNC_TRACE) serr('[sync] begin import: ')` emitted BEFORE `importFile` (the existing `progress.tick` fires only AFTER `importFile` returns — useless when one file wedges). Off by default to avoid a line per file on huge brains; `serr` is source-prefix-aware so under `--workers >1` / `--all` the stuck file is the begin-line with no matching completion. Triage doc: `docs/architecture/serve-sync-concurrency.md` (PGLite single-writer serve↔sync contention — stop `gbrain serve` before a large PGLite sync — plus the `GBRAIN_SYNC_TRACE` + `--no-schema-pack` hang-triage recipes). - `src/commands/import.ts` — `gbrain import` CLI + `runImport` library entrypoint. v0.34.2.0 replaces the prior positional-index checkpoint (`processedIndex: N` into a sorted file list) with a path-set checkpoint via `src/core/import-checkpoint.ts`. The walk still applies `sortNewestFirst()` for embed-cost ordering, but checkpoint correctness no longer depends on sort order. A file enters `completed: Set` only when its `processFile` returns success (including content-hash short-circuit no-ops); failed files never enter the set, so the next run retries them automatically with no manual `~/.gbrain/import-checkpoint.json` delete. Three bug classes died: parallel-import-with-slow-worker drops the slow file on crash-resume (closed — the slow file isn't in `completed` until its own `processFile` resolves), failed-file-bumps-counter-past-itself (closed — failures don't add to `completed`), and v0.33.x sort-flip-drops-newest-N-on-cross-version-resume (closed — order is no longer part of the checkpoint). Old positional checkpoints are detected and discarded with a stderr line on first resume; re-walking is cheap because `content_hash` short-circuits unchanged files. Checkpoint persists every 100 successful adds, not every 100 processed files, so a long failure tail doesn't churn the JSON. Pinned by `test/import-checkpoint.test.ts` (18 unit cases over the helpers) + `test/import-resume.test.ts` (5 integration cases under PGLite, including the SLUG_MISMATCH retry regression codex caught during plan-eng-review). - `src/core/import-checkpoint.ts` (v0.34.2.0) — `loadCheckpoint(brainDir)`, `saveCheckpoint(brainDir, completed)`, `resumeFilter(files, completed, brainDir)`, `clearCheckpoint()`, plus the `ImportCheckpoint` type. Path-set checkpoint format (`{schema_version, brainDir, completed: string[]}`) replaces the v0.33.x positional `{processedIndex: N}` format. Atomic write via `.tmp` + `rename()` so a mid-write crash never leaves a partial JSON. `loadCheckpoint` returns `null` on: missing file, malformed JSON, brainDir mismatch (you ran import against a different brain), and the old positional format (logged to stderr before being discarded). `resumeFilter` returns `{toProcess, skippedCount}` — pure, no I/O, deterministic. `clearCheckpoint` is no-op-on-missing for clean-exit cleanup. Honors `GBRAIN_HOME` via `gbrainPath()` so test isolation via `withEnv({GBRAIN_HOME: tmpdir})` works without monkey-patching the fs layer. Best-effort persistence — `saveCheckpoint` logs warnings on write errors but never throws, so import keeps making progress even if disk is full. - `src/core/sort-newest-first.ts` (v0.34.2.0) — single source of truth for the descending-lex sort that `gbrain import` and `gbrain sync` both apply. Mutates in place (Array.prototype.sort semantics), returns the same array reference for fluent chaining. Empty/single-element inputs short-circuit. Future ordering changes flip one line in this helper instead of touching two CLI commands. Pinned by `test/sort-newest-first.test.ts` (5 hermetic cases: descending order, mixed prefixes, empty input, single-element input, in-place-mutation contract). @@ -765,7 +765,8 @@ Key files (v0.40.7.0 additions): - `src/core/schema-pack/mutate-audit.ts` — ISO-week JSONL at `~/.gbrain/audit/schema-mutations-YYYY-Www.jsonl`. Privacy-redacted per D20: type names → sha8, prefixes → first slug segment only, matches `candidate-audit.ts` privacy posture. Logs BOTH success AND failure events so the v0.40.7+ `schema_pack_writability` doctor check has signal to read. `summarizeMutations()` is the cross-surface parity primitive. - `src/core/schema-pack/registry.ts` extensions — `invalidatePackCache(name?)` walks the extends-chain reverse-graph (codex C6 fix; pre-v0.40.7, editing a parent pack silently left children stale). `tryCachedPack(name)` TTL-gated fast path: inside `STAT_TTL_MS` (default 1000ms, env `GBRAIN_PACK_STAT_TTL_MS`) returns cached without statting. Outside the window: stats every file in the chain; cascade-invalidates on mtime change (D11 cross-process detection). - `src/core/schema-pack/best-effort.ts` — `loadActivePackBestEffort(ctx)` returns `ResolvedPack | null`. Single source of truth for the T1.5 wiring sites. `null` means EMPTY FILTER (NOT hardcoded defaults — D4 contract closing the silent-violation bug class). -- `src/core/schema-pack/lint-rules.ts` — 11 pure rule functions. `withMutation`'s pre-write validation gate composes the 9 file-plane rules; the 2 DB-aware rules (`extractable_empty_corpus`, `mutation_count_anomaly`) need an engine. Single source of truth consumed by CLI lint + MCP `schema_lint` + the pre-write validation gate. +- `src/core/schema-pack/lint-rules.ts` — 12 pure rule functions. `withMutation`'s pre-write validation gate composes the 10 file-plane rules; the 2 DB-aware rules (`extractable_empty_corpus`, `mutation_count_anomaly`) need an engine. Single source of truth consumed by CLI lint + MCP `schema_lint` + the pre-write validation gate. **v0.41.37.0 (#1569):** new file-plane rule `link_regex_catastrophic_backtrack` — advisory ReDoS pre-screen that flags the classic nested-quantifier shapes (`(a+)+`, `(a*)*`, `(a+)*`, `(\w+)+`) in a link_type's `inference.regex` via `NESTED_QUANTIFIER_RE`. WARNING not error: a hard reject would disable the whole pack on upgrade (pages fall back to legacy typing). The runtime input-length cap in `redos-guard.ts` is the actual safety net; this rule tells the pack author to fix the pattern. +- `src/core/schema-pack/redos-guard.ts` + `src/core/schema-pack/link-inference.ts` (v0.41.37.0, #1569) — ReDoS hardening for pack inference regexes. `redos-guard.ts` adds `MAX_REGEX_INPUT_CHARS` (default 64_000, env override `GBRAIN_MAX_REGEX_INPUT_CHARS`) — a hard input-length cap, the real runtime safety net (catastrophic backtracking needs a long input to blow up; a link-extraction `context` is normally a sentence or short paragraph). Over the cap, `runRegexBounded` throws the new tagged `RegexInputTooLargeError` and the regex is skipped (degrade-to-mentions) without entering the `node:vm`. `link-inference.ts:inferLinkTypeFromPack` no-budget branch (test contexts) previously ran `new RegExp(pattern).test(context)` UNBOUNDED — the one ReDoS hole with no timeout; it now routes through `runRegexBounded` so the input-length cap + per-regex vm timeout (`PER_REGEX_TIMEOUT_MS = 50`) apply on every path. Defensive hardening + diagnostics; the deterministic ~3100-file sync-wedge root cause remains open (no repro). Pinned by `test/redos-hardening.test.ts` + `test/schema-pack-lint-rules.test.ts`. - `src/core/schema-pack/query-cache-invalidator.ts` — `invalidateQueryCache(engine, sourceId?)` DELETEs query_cache rows so cached search results bound to old page types don't survive a schema mutation. Codex C9 fix. - `src/core/schema-pack/mutate.ts` — 8-step `withMutation` skeleton (bundled-guard → lock → read → mutator → validate → atomic write → audit → invalidate). 11 mutation primitives: `addTypeToPack`, `removeTypeFromPack` (with codex C14 reference check), `updateTypeOnPack`, `addAliasToType`, `removeAliasFromType`, `addPrefixToType`, `removePrefixFromType`, `addLinkTypeToPack`, `removeLinkTypeFromPack`, `setExtractableOnType`, `setExpertRoutingOnType`. Atomic write via `.tmp + fsync + rename` — pack file on disk is NEVER partial. Inline minimal JSON→YAML emitter so YAML packs stay YAML (does NOT preserve comments — pin pack.json if you care about layout). - `src/core/schema-pack/stats.ts` — `runStatsCore(engine, opts)` returns per-source + aggregate page counts + coverage % + `dead_prefixes` (declared prefixes with zero matching pages — agent's drilldown signal). Multi-source aware (`sourceIds[]` federated, `sourceId` single, or whole-brain). PGLite + Postgres parity via `executeRaw`. Empty brain → coverage:1.0 (vacuous truth). @@ -3009,6 +3010,53 @@ anthropic/claude-sonnet-4-6 --max-cost 5` failed with matched the colon form. Both shapes work now. No config change, no schema migration — `gbrain upgrade` is the whole fix. +**`gbrain reindex --markdown` wiped your auto/dream/signal-detector +tags?** v0.41.37.0 makes tag reconciliation add-only. Re-import and +`reindex --markdown` now ADD current frontmatter tags and never delete, +so enrichment tags written to the DB (auto-tag, dream synthesize, +signal-detector) survive a re-chunk. The reindex DB-only fallback also +reconstructs the full markdown (frontmatter + body + timeline) before +re-chunking, so a page with no on-disk source keeps its frontmatter, +title, and timeline instead of getting overwritten with empty +frontmatter. Trade-off: removing a tag from a page's frontmatter no +longer removes it from the DB on the next sync (frontmatter-tag removal +needs a provenance column, deferred). (Closes #1621.) + +**`gbrain sync` wedges on a large brain (no progress, high CPU)?** +v0.41.37.0 ships three things. First, name the stalling file: + +```bash +GBRAIN_SYNC_TRACE=1 gbrain sync --no-pull --no-embed --yes +``` + +The last `[sync] begin import: ` line with no following completion +is the file being processed when the hang hit. Second, if you suspect a +schema-pack `inference.regex` with catastrophic backtracking, complete +the sync with the pack disabled and re-run extraction later: + +```bash +gbrain sync --no-schema-pack --no-pull --no-embed --yes +``` + +`gbrain schema lint` now warns on the classic nested-quantifier ReDoS +shapes (`(a+)+`, `(a*)*`, …) in pack regexes, and the runtime caps +inference-regex input length (override via `GBRAIN_MAX_REGEX_INPUT_CHARS`). +Third, on a PGLite brain, stop `gbrain serve` before a large sync — +PGLite is single-writer and a live MCP server contends for the write +lock. See [`docs/architecture/serve-sync-concurrency.md`](docs/architecture/serve-sync-concurrency.md) +for the full triage. (Closes #1569.) + +**`gbrain init --migrate-only` / a schema migration fails on Windows +with `getaddrinfo ENOTFOUND`?** v0.41.37.0 runs the 9 schema-bring-up +phases in-process instead of spawning a child `gbrain init +--migrate-only` per phase. The spawned child died on +Windows + bun + Supabase pooler with a DNS-resolution failure even +though the parent connected fine; running in-process removes the spawn +entirely. The v0.13.1 grandfather migration that hung 70+ minutes on an +82K-page PGLite brain is also fixed — it now runs as a chunked bulk SQL +pass (keyed on the page PK, soft-delete-filtered, source-safe) that +completes in ~1-2 seconds. (Closes #1605, #1581.) + ## Docs - [`docs/INSTALL.md`](docs/INSTALL.md) — every install path, end to end diff --git a/package.json b/package.json index 12adeb4b0..ad23780bd 100644 --- a/package.json +++ b/package.json @@ -141,5 +141,5 @@ "bun": ">=1.3.10" }, "license": "MIT", - "version": "0.41.36.0" + "version": "0.41.37.0" } diff --git a/scripts/llms-config.ts b/scripts/llms-config.ts index 1c41c0ac9..8365216a4 100644 --- a/scripts/llms-config.ts +++ b/scripts/llms-config.ts @@ -255,9 +255,10 @@ export const INLINE_TIPS = [ "`gbrain upgrade` runs post-upgrade + apply-migrations.", ]; -// Target ~700KB so llms-full.txt fits in ~175k-token contexts with room to spare. -// Bumped from 600KB in v0.41.9.0 — CLAUDE.md grew past 600KB after the wave's -// new-file annotations + Conductor branch-name iron-rule landed; the bundle -// still fits comfortably in modern long-context models. +// Target so llms-full.txt fits in ~190k-token contexts with room to spare. +// Bumped 600KB→700KB in v0.41.9.0, then 700KB→750KB in v0.41.37.0 — CLAUDE.md +// crossed 700KB once the critical-fix-wave key-files annotations merged on top +// of master's v0.41.34/35/36 additions (703KB). Still fits comfortably in +// modern long-context models. // Generator prints a WARN if exceeded; ship with includeInFull=false exclusions. -export const FULL_SIZE_BUDGET = 700_000; +export const FULL_SIZE_BUDGET = 750_000; diff --git a/src/commands/init.ts b/src/commands/init.ts index e2d7e7daa..0e14c4dc7 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -515,44 +515,27 @@ async function resolveChatByEnv(out: ResolvedAIOptions): Promise { * clobbering the user's chosen engine. */ async function initMigrateOnly(opts: { jsonOutput: boolean }) { - const config = loadConfig(); - if (!config) { - const msg = 'No brain configured. Run `gbrain init` (interactive) or `gbrain init --pglite` / `gbrain init --supabase` first.'; + // v0.41.37.0 #1605: delegate to the shared runMigrateOnlyCore so the CLI path + // and the in-process migration-orchestrator path can't drift (single source + // of truth for configureGateway-before-initSchema + the schema bring-up). + const { runMigrateOnlyCore, MigrateOnlyError } = await import('./migrations/in-process.ts'); + try { + const result = await runMigrateOnlyCore(); if (opts.jsonOutput) { - console.log(JSON.stringify({ status: 'error', reason: 'no_config', message: msg })); + console.log(JSON.stringify({ status: 'success', engine: result.engine, mode: 'migrate-only' })); + } else { + console.log(`Schema up to date (engine: ${result.engine}).`); + } + } catch (e) { + const isNoConfig = e instanceof MigrateOnlyError && e.message.startsWith('No brain configured'); + const msg = e instanceof Error ? e.message : String(e); + if (opts.jsonOutput) { + console.log(JSON.stringify({ status: 'error', reason: isNoConfig ? 'no_config' : 'migrate_failed', message: msg })); } else { console.error(msg); } process.exit(1); } - - // B.3: configureGateway BEFORE initSchema even on the migrate-only path, - // so a schema bump on a brain whose file config is missing the embedding - // fields doesn't fall through to stale hardcoded fallbacks. Reads - // existing config (which loadConfig already merged with env) and - // propagates it into the gateway. - const { configureGateway: configureGw } = await import('../core/ai/gateway.ts'); - configureGw({ - embedding_model: config.embedding_model, - embedding_dimensions: config.embedding_dimensions, - expansion_model: config.expansion_model, - chat_model: config.chat_model, - env: { ...process.env }, - }); - - const engine = await createEngine(toEngineConfig(config)); - try { - await engine.connect(toEngineConfig(config)); - await engine.initSchema(); - } finally { - try { await engine.disconnect(); } catch { /* best-effort */ } - } - - if (opts.jsonOutput) { - console.log(JSON.stringify({ status: 'success', engine: config.engine, mode: 'migrate-only' })); - } else { - console.log(`Schema up to date (engine: ${config.engine}).`); - } } /** diff --git a/src/commands/migrations/in-process.ts b/src/commands/migrations/in-process.ts new file mode 100644 index 000000000..784fea3df --- /dev/null +++ b/src/commands/migrations/in-process.ts @@ -0,0 +1,139 @@ +/** + * In-process migration helpers (v0.41.37.0 #1605). + * + * Why this exists: migration schema phases used to shell out to a child + * `gbrain init --migrate-only` via `execSync`. On Windows + bun + Supabase + * pooler, the spawned CHILD process dies with `getaddrinfo ENOTFOUND` before it + * can connect — even though the PARENT connects fine and `env: process.env` is + * passed. It is a bun-on-Windows child-process DNS-resolution failure, not an + * env-propagation bug. The only robust fix is to not spawn at all: run the + * schema bring-up IN-PROCESS. The PGLite path at v0_11_0.ts already proved the + * pattern; this generalizes it to every engine + every schema phase. + * + * `runMigrateOnlyCore` is the single source of truth for "bring schema to head" + * — `init.ts:initMigrateOnly` (the `gbrain init --migrate-only` CLI path) and + * the migration orchestrators both call it, so the configureGateway-before- + * initSchema fix can't drift between them. + * + * `runGbrainSubprocess` is the diagnostic wrapper for the REMAINING (non-schema) + * gbrain-subprocess spawns (extract/repair/stats). It captures child stderr and + * folds it into the thrown error so a Windows failure shows the real + * `getaddrinfo ENOTFOUND` line instead of the bare `Command failed: ...`. + */ + +import { execSync } from 'child_process'; + +import { loadConfig, toEngineConfig } from '../../core/config.ts'; +import { createEngine } from '../../core/engine-factory.ts'; + +/** Default wall-clock guard for in-process initSchema. Matches the 600s cap + * the old `execSync('gbrain init --migrate-only', { timeout: 600_000 })` used, + * so a hung schema bring-up surfaces as a phase failure instead of wedging + * the whole cascade. */ +export const MIGRATE_ONLY_TIMEOUT_MS = 600_000; + +/** Large stderr buffer for captured subprocess output. `execSync`'s default + * ~1MB maxBuffer overflows on long backfills (extract/repair) and turns a + * successful run into a spurious failure. */ +const SUBPROCESS_MAX_BUFFER = 64 * 1024 * 1024; + +export interface MigrateOnlyResult { + /** The engine kind that was brought to head ('pglite' | 'postgres'). */ + engine: string; +} + +export class MigrateOnlyError extends Error { + constructor(message: string) { + super(message); + this.name = 'MigrateOnlyError'; + } +} + +/** + * Bring the configured brain's schema to head, in-process. Mirrors what + * `gbrain init --migrate-only` did via subprocess: configureGateway → + * createEngine → connect → initSchema → disconnect. Idempotent (initSchema is + * a no-op when already at head). Throws `MigrateOnlyError` on no-config or + * timeout so callers report a failed phase rather than hanging. + */ +export async function runMigrateOnlyCore(opts?: { timeoutMs?: number }): Promise { + const config = loadConfig(); + if (!config) { + throw new MigrateOnlyError( + 'No brain configured. Run `gbrain init` (interactive) or `gbrain init --pglite` / `gbrain init --supabase` first.', + ); + } + + // configureGateway BEFORE initSchema (init.ts B.3): a schema bump on a brain + // whose file config is missing embedding fields must not fall through to + // stale hardcoded fallbacks. loadConfig already merged env; propagate it. + const { configureGateway } = await import('../../core/ai/gateway.ts'); + configureGateway({ + embedding_model: config.embedding_model, + embedding_dimensions: config.embedding_dimensions, + expansion_model: config.expansion_model, + chat_model: config.chat_model, + env: { ...process.env }, + }); + + const timeoutMs = opts?.timeoutMs ?? MIGRATE_ONLY_TIMEOUT_MS; + const engine = await createEngine(toEngineConfig(config)); + try { + await engine.connect(toEngineConfig(config)); + await withTimeout( + engine.initSchema(), + timeoutMs, + `schema init timed out after ${Math.round(timeoutMs / 1000)}s`, + ); + } finally { + try { await engine.disconnect(); } catch { /* best-effort */ } + } + + return { engine: config.engine }; +} + +/** + * Run a `gbrain ...` subcommand as a subprocess, capturing child stderr so a + * failure surfaces the real reason. Used for the non-schema backfill phases + * (extract/repair/stats) that aren't yet in-process. On Windows these may still + * fail with `getaddrinfo ENOTFOUND`, but the operator now sees WHY instead of a + * bare `Command failed`. Returns captured stdout (utf-8) on success. + * + * Note: stderr is piped (captured), so gbrain progress lines (which go to + * stderr) are not shown live during these phases — acceptable for a one-shot + * `apply-migrations` run; the failure reason matters more than live progress. + */ +export function runGbrainSubprocess(cmd: string, opts?: { timeoutMs?: number }): string { + try { + const out = execSync(cmd, { + stdio: ['inherit', 'pipe', 'pipe'], + timeout: opts?.timeoutMs ?? MIGRATE_ONLY_TIMEOUT_MS, + env: process.env, + maxBuffer: SUBPROCESS_MAX_BUFFER, + encoding: 'utf-8', + }); + return typeof out === 'string' ? out : ''; + } catch (e: unknown) { + const err = e as { message?: string; stderr?: Buffer | string }; + const stderrRaw = err?.stderr + ? (Buffer.isBuffer(err.stderr) ? err.stderr.toString('utf-8') : String(err.stderr)) + : ''; + const tail = stderrRaw.split('\n').filter(Boolean).slice(-10).join('\n'); + const base = err?.message ?? String(e); + throw new Error(tail ? `${base}\n--- child stderr (tail) ---\n${tail}` : base); + } +} + +/** Reject `p` if it doesn't settle within `ms`. The original promise keeps + * running (best-effort) but the caller sees a clear timeout error. */ +async function withTimeout(p: Promise, ms: number, message: string): Promise { + let timer: ReturnType | undefined; + const timeout = new Promise((_, reject) => { + timer = setTimeout(() => reject(new MigrateOnlyError(message)), ms); + }); + try { + return await Promise.race([p, timeout]); + } finally { + if (timer) clearTimeout(timer); + } +} diff --git a/src/commands/migrations/v0_11_0.ts b/src/commands/migrations/v0_11_0.ts index e9c3ee685..db75df6f1 100644 --- a/src/commands/migrations/v0_11_0.ts +++ b/src/commands/migrations/v0_11_0.ts @@ -23,7 +23,6 @@ import { existsSync, readFileSync, writeFileSync, mkdirSync, appendFileSync, lstatSync, statSync, realpathSync } from 'fs'; import { join, resolve, dirname } from 'path'; import { execSync } from 'child_process'; -import { childGlobalFlags } from '../../core/cli-options.ts'; import type { Migration, OrchestratorOpts, OrchestratorResult, OrchestratorPhaseResult } from './types.ts'; import { savePreferences, loadPreferences } from '../../core/preferences.ts'; // Bug 3 — appendCompletedMigration moved to the runner (apply-migrations.ts). @@ -61,28 +60,14 @@ export interface PendingHostWorkEntry { async function phaseASchema(opts: OrchestratorOpts): Promise { if (opts.dryRun) return { name: 'schema', status: 'skipped', detail: 'dry-run' }; try { - // v0.36.x #1100: route PGLite through an in-process schema apply rather - // than `execSync('gbrain init --migrate-only')`. The subprocess inherits - // HOME and tries to acquire the same file lock the parent process is - // holding (or briefly released and the on-disk artifact has not finished - // settling), which deadlocks until the 30s lock timeout fires. The - // structural fix is to not spawn a subprocess for work the parent can - // do directly — Postgres tolerates concurrent connections, so the - // legacy execSync path stays for Postgres callers. - const { loadConfig, toEngineConfig } = await import('../../core/config.ts'); - const cfg = loadConfig(); - if (cfg?.engine === 'pglite') { - const { createEngine } = await import('../../core/engine-factory.ts'); - const eng = await createEngine(toEngineConfig(cfg)); - try { - await eng.connect(toEngineConfig(cfg)); - await eng.initSchema(); - } finally { - try { await eng.disconnect(); } catch { /* best-effort */ } - } - return { name: 'schema', status: 'complete' }; - } - execSync('gbrain init --migrate-only' + childGlobalFlags(), { stdio: 'inherit', timeout: 60_000, env: process.env }); + // v0.41.37.0 #1605: bring schema to head IN-PROCESS for every engine. Was an + // a `gbrain init --migrate-only` subprocess subprocess for Postgres (died with + // `getaddrinfo ENOTFOUND` on Windows+bun+Supabase-pooler before it could + // connect) plus a PGLite-only in-process branch (which separately + // deadlocked on the file lock, #1100). runMigrateOnlyCore is the single + // in-process path for both engines. + const { runMigrateOnlyCore } = await import('./in-process.ts'); + await runMigrateOnlyCore(); return { name: 'schema', status: 'complete' }; } catch (e) { const msg = e instanceof Error ? e.message : String(e); diff --git a/src/commands/migrations/v0_12_0.ts b/src/commands/migrations/v0_12_0.ts index 0bbe82d3f..4aa64b9b3 100644 --- a/src/commands/migrations/v0_12_0.ts +++ b/src/commands/migrations/v0_12_0.ts @@ -31,19 +31,21 @@ */ import { execSync } from 'child_process'; +import { runGbrainSubprocess } from './in-process.ts'; import type { Migration, OrchestratorOpts, OrchestratorResult, OrchestratorPhaseResult } from './types.ts'; import { childGlobalFlags } from '../../core/cli-options.ts'; // Bug 3 — ledger writes moved to the runner (apply-migrations.ts). // ── Phase A — Schema ──────────────────────────────────────── -function phaseASchema(opts: OrchestratorOpts): OrchestratorPhaseResult { +async function phaseASchema(opts: OrchestratorOpts): Promise { if (opts.dryRun) return { name: 'schema', status: 'skipped', detail: 'dry-run' }; try { // 10-minute budget. Migrations v8/v9 dedup with helper-index should be sub-second // even on 80K-duplicate brains, but the outer wall-clock cap shouldn't be the // failure mode (the prior 60s ceiling tripped Garry's production upgrade). - execSync('gbrain init --migrate-only' + childGlobalFlags(), { stdio: 'inherit', timeout: 600_000, env: process.env }); + const { runMigrateOnlyCore } = await import('./in-process.ts'); + await runMigrateOnlyCore(); return { name: 'schema', status: 'complete' }; } catch (e) { const msg = e instanceof Error ? e.message : String(e); @@ -93,7 +95,7 @@ function phaseCBackfillLinks(opts: OrchestratorOpts): OrchestratorPhaseResult { // --source db is idempotent: the UNIQUE constraint on // (from_page_id, to_page_id, link_type) and ON CONFLICT DO NOTHING // make re-runs cheap. Empty brains return 0/0 quickly. - execSync('gbrain extract links --source db' + childGlobalFlags(), { stdio: 'inherit', timeout: 600_000, env: process.env }); + runGbrainSubprocess('gbrain extract links --source db' + childGlobalFlags(), { timeoutMs: 600_000 }); return { name: 'backfill_links', status: 'complete' }; } catch (e) { const msg = e instanceof Error ? e.message : String(e); @@ -104,7 +106,7 @@ function phaseCBackfillLinks(opts: OrchestratorOpts): OrchestratorPhaseResult { function phaseDBackfillTimeline(opts: OrchestratorOpts): OrchestratorPhaseResult { if (opts.dryRun) return { name: 'backfill_timeline', status: 'skipped', detail: 'dry-run' }; try { - execSync('gbrain extract timeline --source db' + childGlobalFlags(), { stdio: 'inherit', timeout: 600_000, env: process.env }); + runGbrainSubprocess('gbrain extract timeline --source db' + childGlobalFlags(), { timeoutMs: 600_000 }); return { name: 'backfill_timeline', status: 'complete' }; } catch (e) { const msg = e instanceof Error ? e.message : String(e); @@ -188,7 +190,7 @@ async function orchestrator(opts: OrchestratorOpts): Promise const phases: OrchestratorPhaseResult[] = []; // A. Schema - const a = phaseASchema(opts); + const a = await phaseASchema(opts); phases.push(a); if (a.status === 'failed') { return finalizeResult(phases, 'failed'); diff --git a/src/commands/migrations/v0_12_2.ts b/src/commands/migrations/v0_12_2.ts index 9400866ce..1902566dd 100644 --- a/src/commands/migrations/v0_12_2.ts +++ b/src/commands/migrations/v0_12_2.ts @@ -21,18 +21,20 @@ */ import { execSync } from 'child_process'; +import { runGbrainSubprocess } from './in-process.ts'; import type { Migration, OrchestratorOpts, OrchestratorResult, OrchestratorPhaseResult } from './types.ts'; import { childGlobalFlags } from '../../core/cli-options.ts'; // Bug 3 — ledger writes moved to the runner (apply-migrations.ts). // ── Phase A — Schema ──────────────────────────────────────── -function phaseASchema(opts: OrchestratorOpts): OrchestratorPhaseResult { +async function phaseASchema(opts: OrchestratorOpts): Promise { if (opts.dryRun) return { name: 'schema', status: 'skipped', detail: 'dry-run' }; try { // Propagate global progress flags so the child shows the same mode the // parent orchestrator is running in. - execSync('gbrain init --migrate-only' + childGlobalFlags(), { stdio: 'inherit', timeout: 60_000, env: process.env }); + const { runMigrateOnlyCore } = await import('./in-process.ts'); + await runMigrateOnlyCore(); return { name: 'schema', status: 'complete' }; } catch (e) { const msg = e instanceof Error ? e.message : String(e); @@ -46,7 +48,7 @@ function phaseBRepair(opts: OrchestratorOpts): OrchestratorPhaseResult { if (opts.dryRun) return { name: 'jsonb_repair', status: 'skipped', detail: 'dry-run' }; try { // stdio: 'inherit' — child's stderr progress streams straight through. - execSync('gbrain repair-jsonb' + childGlobalFlags(), { stdio: 'inherit', timeout: 600_000, env: process.env }); + runGbrainSubprocess('gbrain repair-jsonb' + childGlobalFlags(), { timeoutMs: 600_000 }); return { name: 'jsonb_repair', status: 'complete' }; } catch (e) { const msg = e instanceof Error ? e.message : String(e); @@ -94,7 +96,7 @@ async function orchestrator(opts: OrchestratorOpts): Promise const phases: OrchestratorPhaseResult[] = []; - const a = phaseASchema(opts); + const a = await phaseASchema(opts); phases.push(a); if (a.status === 'failed') return finalizeResult(phases, 'failed'); diff --git a/src/commands/migrations/v0_13_0.ts b/src/commands/migrations/v0_13_0.ts index b23ec4081..68510339b 100644 --- a/src/commands/migrations/v0_13_0.ts +++ b/src/commands/migrations/v0_13_0.ts @@ -26,6 +26,7 @@ */ import { execSync } from 'child_process'; +import { runGbrainSubprocess } from './in-process.ts'; import type { Migration, OrchestratorOpts, OrchestratorResult, OrchestratorPhaseResult } from './types.ts'; // Bug 3 — ledger writes moved to the runner (apply-migrations.ts). The // orchestrator returns its result and the runner persists it. @@ -44,10 +45,11 @@ import type { Migration, OrchestratorOpts, OrchestratorResult, OrchestratorPhase // upgrade mid-migration. The shim is already the canonical wrapper; trust // it. Regression guarded by test/migrations-v0_13_0.test.ts. -function phaseASchema(opts: OrchestratorOpts): OrchestratorPhaseResult { +async function phaseASchema(opts: OrchestratorOpts): Promise { if (opts.dryRun) return { name: 'schema', status: 'skipped', detail: 'dry-run' }; try { - execSync('gbrain init --migrate-only', { stdio: 'inherit', timeout: 600_000, env: process.env }); + const { runMigrateOnlyCore } = await import('./in-process.ts'); + await runMigrateOnlyCore(); return { name: 'schema', status: 'complete' }; } catch (e) { const msg = e instanceof Error ? e.message : String(e); @@ -64,11 +66,7 @@ function phaseBBackfill(opts: OrchestratorOpts): OrchestratorPhaseResult { // `--include-frontmatter` is the v0.13 flag that enables the canonical // frontmatter link extractor. Default-OFF in the CLI for back-compat; // the migration explicitly opts in because this is the canonical backfill. - execSync('gbrain extract links --source db --include-frontmatter', { - stdio: 'inherit', - timeout: 1_800_000, // 30 min hard cap; typical 2-5 min on 46K pages - env: process.env, - }); + runGbrainSubprocess('gbrain extract links --source db --include-frontmatter', { timeoutMs: 1_800_000 }); return { name: 'frontmatter_backfill', status: 'complete' }; } catch (e) { const msg = e instanceof Error ? e.message : String(e); @@ -116,7 +114,7 @@ async function orchestrator(opts: OrchestratorOpts): Promise const phases: OrchestratorPhaseResult[] = []; - const a = phaseASchema(opts); + const a = await phaseASchema(opts); phases.push(a); if (a.status === 'failed') return finalizeResult(phases, 'failed'); diff --git a/src/commands/migrations/v0_13_1.ts b/src/commands/migrations/v0_13_1.ts index a3c479d2d..40672b965 100644 --- a/src/commands/migrations/v0_13_1.ts +++ b/src/commands/migrations/v0_13_1.ts @@ -20,13 +20,14 @@ * frontmatter snapshot. Roll back by re-applying those snapshots via * `gbrain apply-migrations --rollback v0.13.0` (future CLI; not in scope). * - * Scale: on a 30K-page brain, ~15s on Postgres, ~30s on PGLite. Batched in - * chunks of 100 with a commit per batch so interruption losses are bounded. + * Scale (v0.41.37.0 #1581): chunked bulk SQL — one id-snapshot SELECT + N + * (SELECT-for-rollback + UPDATE) statements of CHUNK_SIZE rows each. Completes + * in ~1-2s even on an 82K-page PGLite brain. The prior per-page + * getPage+putPage loop hung CPU-bound for 70+ min on that brain (#1581). * - * Snapshot-slugs rule: reads engine.getAllSlugs() upfront into an in-memory - * Set before iterating. Prior learning [listpages-pagination-mutation]: any - * batch write that mutates updated_at during OFFSET pagination is unstable. - * getAllSlugs returns a full snapshot that isn't invalidated by our writes. + * Snapshot rule: the affected id set is read once via SQL up front. It isn't + * invalidated by our writes because each UPDATE flips its rows out of the + * GRANDFATHER_WHERE predicate (idempotent + resumable). * * Safety: does NOT call saveConfig. Prior learning [gbrain-init-default-pglite-flip]: * bare `gbrain init` defaults to PGLite and overwrites Postgres config. @@ -46,7 +47,6 @@ import type { BrainEngine } from '../../core/engine.ts'; // Lazy: GBRAIN_HOME may be set after module load. const getRollbackDir = () => gbrainPath('migrations'); const getRollbackFile = () => join(getRollbackDir(), 'v0_13_1-rollback.jsonl'); -const BATCH_SIZE = 100; // --------------------------------------------------------------------------- // Phase A — connect (no config write) @@ -75,30 +75,35 @@ async function phaseAConnect(opts: OrchestratorOpts): Promise<{ result: Orchestr } } -// --------------------------------------------------------------------------- -// Phase B — snapshot slugs upfront -// --------------------------------------------------------------------------- - -async function phaseBSnapshot(engine: BrainEngine): Promise<{ result: OrchestratorPhaseResult; slugs: string[] }> { - try { - const slugSet = await engine.getAllSlugs(); - const slugs = [...slugSet].sort(); - return { - result: { name: 'snapshot', status: 'complete', detail: `${slugs.length} slugs` }, - slugs, - }; - } catch (e) { - return { - result: { name: 'snapshot', status: 'failed', detail: e instanceof Error ? e.message : String(e) }, - slugs: [], - }; - } -} - // --------------------------------------------------------------------------- // Phase C — grandfather: add validate:false where absent +// +// v0.41.37.0 #1581: rewritten from a per-page getPage+putPage loop (which hung +// CPU-bound for 70+ min on an 82K-page PGLite brain) to a CHUNKED bulk SQL +// pass. Three correctness properties (the per-page loop and a naive single +// bulk UPDATE both got these wrong): +// 1. Keyed on pages.id (globally unique PK), NOT slug. `slug` uniqueness is +// (source_id, slug) — a slug-batched UPDATE would mutate same-slug pages +// across other sources and produce ambiguous rollback rows. +// 2. Filters `deleted_at IS NULL` — the old getPage path hid soft-deleted +// rows; a raw UPDATE must not grandfather tombstones. +// 3. Chunked in CHUNK_SIZE batches so lock-hold stays bounded (the +// DELETE_BATCH_SIZE convention) instead of one giant transaction. +// The rollback log carries source identity ({id, slug, source_id, +// pre_frontmatter}) so a rollback is unambiguous across sources. // --------------------------------------------------------------------------- +// Filter for pages still needing the grandfather flag. Literal `?` jsonb +// existence operator (matches src/core/embed-skip.ts convention); 'validate' +// is a hardcoded literal, no injection surface. COALESCE for null-frontmatter +// safety. deleted_at IS NULL skips soft-deleted tombstones. +const GRANDFATHER_WHERE = + "NOT (COALESCE(frontmatter, '{}'::jsonb) ? 'validate') AND deleted_at IS NULL"; + +// Per-chunk row count. Bounded lock-hold + write-amplification per statement +// (same rationale as engine-constants.ts DELETE_BATCH_SIZE). +const CHUNK_SIZE = 1000; + interface GrandfatherResult { touched: number; skipped: number; @@ -106,60 +111,69 @@ interface GrandfatherResult { failures: string[]; } -async function phaseCGrandfather( +// Exported for direct hermetic testing against a PGLite engine (the config / +// loadConfig flow is exercised separately). Internal helper otherwise. +export async function phaseCGrandfather( engine: BrainEngine, - slugs: string[], opts: OrchestratorOpts, ): Promise<{ result: OrchestratorPhaseResult; detail: GrandfatherResult }> { - ensureRollbackDir(); const gf: GrandfatherResult = { touched: 0, skipped: 0, failed: 0, failures: [] }; - for (let i = 0; i < slugs.length; i += BATCH_SIZE) { - const batch = slugs.slice(i, i + BATCH_SIZE); - for (const slug of batch) { + try { + if (opts.dryRun) { + const rows = await engine.executeRaw<{ count: string | number }>( + `SELECT COUNT(*) AS count FROM pages WHERE ${GRANDFATHER_WHERE}`, + ); + const c = rows[0]?.count ?? 0; + gf.touched = typeof c === 'string' ? parseInt(c, 10) : Number(c); + return { + result: { name: 'grandfather', status: 'complete', detail: `would touch ${gf.touched} (dry-run)` }, + detail: gf, + }; + } + + ensureRollbackDir(); + + // Snapshot the affected id set up front (ids only — cheap, no frontmatter + // blobs in memory). The snapshot isn't invalidated by our writes because + // each UPDATE flips the rows out of the GRANDFATHER_WHERE predicate. + const idRows = await engine.executeRaw<{ id: number }>( + `SELECT id FROM pages WHERE ${GRANDFATHER_WHERE} ORDER BY id`, + ); + const ids = idRows.map(r => Number(r.id)); + + for (let i = 0; i < ids.length; i += CHUNK_SIZE) { + const chunk = ids.slice(i, i + CHUNK_SIZE); try { - const page = await engine.getPage(slug); - if (!page) { gf.skipped++; continue; } + // Rollback log BEFORE mutation: one SELECT per chunk (bounded memory), + // one appendFileSync per chunk. Carries source_id so rollback is + // unambiguous across same-slug-different-source pages. + const snap = await engine.executeRaw<{ + id: number; slug: string; source_id: string | null; frontmatter: Record | null; + }>( + 'SELECT id, slug, source_id, frontmatter FROM pages WHERE id = ANY($1::int[])', + [chunk], + ); + appendRollbackBatch(snap); - // Idempotency: skip if frontmatter already has a `validate` key - // (whether true, false, or any other value). We don't flip existing - // explicit settings. - if (page.frontmatter && Object.prototype.hasOwnProperty.call(page.frontmatter, 'validate')) { - gf.skipped++; - continue; - } - - if (opts.dryRun) { - gf.touched++; - continue; - } - - // Rollback log BEFORE mutation, so a crash mid-write still lets us - // revert. Append-only, one line per page, newline-terminated. - appendRollbackEntry({ - slug, - pre_frontmatter: page.frontmatter ?? {}, - }); - - const nextFrontmatter = { ...(page.frontmatter ?? {}), validate: false }; - await engine.putPage(slug, { - type: page.type, - title: page.title, - compiled_truth: page.compiled_truth, - timeline: page.timeline, - frontmatter: nextFrontmatter, - }); - gf.touched++; + await engine.executeRaw( + `UPDATE pages SET frontmatter = jsonb_set(COALESCE(frontmatter, '{}'::jsonb), '{validate}', 'false'::jsonb) ` + + 'WHERE id = ANY($1::int[])', + [chunk], + ); + gf.touched += chunk.length; } catch (e) { - gf.failed++; + gf.failed += chunk.length; const msg = e instanceof Error ? e.message : String(e); - gf.failures.push(`${slug}: ${msg.slice(0, 100)}`); + gf.failures.push(`chunk@${i}: ${msg.slice(0, 100)}`); } } + } catch (e) { + gf.failed += 1; + gf.failures.push(`grandfather: ${e instanceof Error ? e.message : String(e)}`.slice(0, 120)); } - const status: OrchestratorPhaseResult['status'] = - gf.failed > 0 ? 'failed' : 'complete'; + const status: OrchestratorPhaseResult['status'] = gf.failed > 0 ? 'failed' : 'complete'; const detailStr = `touched=${gf.touched} skipped=${gf.skipped} failed=${gf.failed}`; return { result: { name: 'grandfather', status, detail: detailStr }, @@ -215,13 +229,10 @@ async function orchestrator(opts: OrchestratorOpts): Promise } try { - const { result: snapRes, slugs } = await phaseBSnapshot(engine); - phases.push(snapRes); - if (snapRes.status !== 'complete') { - return { version: '0.13.1', status: 'failed', phases }; - } - - const { result: gfRes, detail: gfDetail } = await phaseCGrandfather(engine, slugs, opts); + // v0.41.37.0 #1581: phaseBSnapshot (getAllSlugs) is gone — the chunked bulk + // pass filters via SQL (GRANDFATHER_WHERE), so we no longer materialize a + // full slug list in JS. + const { result: gfRes, detail: gfDetail } = await phaseCGrandfather(engine, opts); phases.push(gfRes); filesRewritten = gfDetail.touched; @@ -255,13 +266,23 @@ function ensureRollbackDir(): void { if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); } -function appendRollbackEntry(entry: { slug: string; pre_frontmatter: Record }): void { - const line = JSON.stringify({ +// v0.41.37.0 #1581: batch rollback writer. One appendFileSync per chunk, bounded +// memory. Each line carries id + slug + source_id so a rollback is unambiguous +// across same-slug-different-source pages (pages.slug is not globally unique). +function appendRollbackBatch( + rows: ReadonlyArray<{ id: number; slug: string; source_id: string | null; frontmatter: Record | null }>, +): void { + if (rows.length === 0) return; + const ts = new Date().toISOString(); + const lines = rows.map(r => JSON.stringify({ migration: 'v0.13.0', - timestamp: new Date().toISOString(), - ...entry, - }) + '\n'; - appendFileSync(getRollbackFile(), line, 'utf-8'); + timestamp: ts, + id: r.id, + slug: r.slug, + source_id: r.source_id ?? 'default', + pre_frontmatter: r.frontmatter ?? {}, + })).join('\n') + '\n'; + appendFileSync(getRollbackFile(), lines, 'utf-8'); } // --------------------------------------------------------------------------- diff --git a/src/commands/migrations/v0_16_0.ts b/src/commands/migrations/v0_16_0.ts index daa7d8179..66b81a1b3 100644 --- a/src/commands/migrations/v0_16_0.ts +++ b/src/commands/migrations/v0_16_0.ts @@ -18,7 +18,6 @@ * C. Record — append completed.jsonl. */ -import { execSync } from 'child_process'; import type { Migration, OrchestratorOpts, OrchestratorResult, OrchestratorPhaseResult } from './types.ts'; import { appendCompletedMigration } from '../../core/preferences.ts'; import { loadConfig, toEngineConfig } from '../../core/config.ts'; @@ -28,10 +27,11 @@ const REQUIRED_TABLES = ['subagent_messages', 'subagent_tool_executions', 'subag // ── Phase A — Schema ──────────────────────────────────────── -function phaseASchema(opts: OrchestratorOpts): OrchestratorPhaseResult { +async function phaseASchema(opts: OrchestratorOpts): Promise { if (opts.dryRun) return { name: 'schema', status: 'skipped', detail: 'dry-run' }; try { - execSync('gbrain init --migrate-only', { stdio: 'inherit', timeout: 60_000, env: process.env }); + const { runMigrateOnlyCore } = await import('./in-process.ts'); + await runMigrateOnlyCore(); return { name: 'schema', status: 'complete' }; } catch (e) { const msg = e instanceof Error ? e.message : String(e); @@ -88,7 +88,7 @@ async function orchestrator(opts: OrchestratorOpts): Promise const phases: OrchestratorPhaseResult[] = []; - const a = phaseASchema(opts); + const a = await phaseASchema(opts); phases.push(a); if (a.status === 'failed') return finalize(phases, 'failed'); diff --git a/src/commands/migrations/v0_18_0.ts b/src/commands/migrations/v0_18_0.ts index a46ca407a..93e8751f7 100644 --- a/src/commands/migrations/v0_18_0.ts +++ b/src/commands/migrations/v0_18_0.ts @@ -19,7 +19,6 @@ * Idempotent: safe to re-run on partial state. */ -import { execSync } from 'child_process'; import type { Migration, OrchestratorOpts, OrchestratorResult, OrchestratorPhaseResult } from './types.ts'; import { appendCompletedMigration } from '../../core/preferences.ts'; import { loadConfig, toEngineConfig } from '../../core/config.ts'; @@ -27,10 +26,11 @@ import { createEngine } from '../../core/engine-factory.ts'; // ── Phase A — Schema ──────────────────────────────────────── -function phaseASchema(opts: OrchestratorOpts): OrchestratorPhaseResult { +async function phaseASchema(opts: OrchestratorOpts): Promise { if (opts.dryRun) return { name: 'schema', status: 'skipped', detail: 'dry-run' }; try { - execSync('gbrain init --migrate-only', { stdio: 'inherit', timeout: 600_000, env: process.env }); + const { runMigrateOnlyCore } = await import('./in-process.ts'); + await runMigrateOnlyCore(); return { name: 'schema', status: 'complete' }; } catch (e) { const msg = e instanceof Error ? e.message : String(e); @@ -174,7 +174,7 @@ async function orchestrator(opts: OrchestratorOpts): Promise const phases: OrchestratorPhaseResult[] = []; - const a = phaseASchema(opts); + const a = await phaseASchema(opts); phases.push(a); if (a.status === 'failed') return finalize(phases, 'failed'); diff --git a/src/commands/migrations/v0_18_1.ts b/src/commands/migrations/v0_18_1.ts index 231f95158..61f057b18 100644 --- a/src/commands/migrations/v0_18_1.ts +++ b/src/commands/migrations/v0_18_1.ts @@ -16,15 +16,15 @@ * fires on upgrade, because doctor + connectEngine never call initSchema(). */ -import { execSync } from 'child_process'; import type { Migration, OrchestratorOpts, OrchestratorResult, OrchestratorPhaseResult } from './types.ts'; // ── Phase A — Schema ──────────────────────────────────────── -function phaseASchema(opts: OrchestratorOpts): OrchestratorPhaseResult { +async function phaseASchema(opts: OrchestratorOpts): Promise { if (opts.dryRun) return { name: 'schema', status: 'skipped', detail: 'dry-run' }; try { - execSync('gbrain init --migrate-only', { stdio: 'inherit', timeout: 600_000, env: process.env }); + const { runMigrateOnlyCore } = await import('./in-process.ts'); + await runMigrateOnlyCore(); return { name: 'schema', status: 'complete' }; } catch (e) { const msg = e instanceof Error ? e.message : String(e); @@ -36,7 +36,7 @@ function phaseASchema(opts: OrchestratorOpts): OrchestratorPhaseResult { async function orchestrator(opts: OrchestratorOpts): Promise { const phases: OrchestratorPhaseResult[] = []; - phases.push(phaseASchema(opts)); + phases.push(await phaseASchema(opts)); const anyFailed = phases.some(p => p.status === 'failed'); const status: OrchestratorResult['status'] = anyFailed ? 'partial' : 'complete'; diff --git a/src/commands/migrations/v0_21_0.ts b/src/commands/migrations/v0_21_0.ts index e35ca6811..d920b3c86 100644 --- a/src/commands/migrations/v0_21_0.ts +++ b/src/commands/migrations/v0_21_0.ts @@ -25,20 +25,15 @@ * All phases are idempotent and safe to re-run. */ -import { execSync } from 'child_process'; import type { Migration, OrchestratorOpts, OrchestratorResult, OrchestratorPhaseResult } from './types.ts'; -import { childGlobalFlags } from '../../core/cli-options.ts'; // ── Phase A — Schema ──────────────────────────────────────── -function phaseASchema(opts: OrchestratorOpts): OrchestratorPhaseResult { +async function phaseASchema(opts: OrchestratorOpts): Promise { if (opts.dryRun) return { name: 'schema', status: 'skipped', detail: 'dry-run' }; try { - execSync('gbrain init --migrate-only' + childGlobalFlags(), { - stdio: 'inherit', - timeout: 600_000, - env: process.env, - }); + const { runMigrateOnlyCore } = await import('./in-process.ts'); + await runMigrateOnlyCore(); return { name: 'schema', status: 'complete' }; } catch (e) { const msg = e instanceof Error ? e.message : String(e); @@ -102,7 +97,7 @@ async function orchestrator(opts: OrchestratorOpts): Promise const phases: OrchestratorPhaseResult[] = []; - const a = phaseASchema(opts); + const a = await phaseASchema(opts); phases.push(a); if (a.status === 'failed') return finalizeResult(phases, 'failed'); diff --git a/src/commands/migrations/v0_29_1.ts b/src/commands/migrations/v0_29_1.ts index ea9c1ed2b..2c22fb494 100644 --- a/src/commands/migrations/v0_29_1.ts +++ b/src/commands/migrations/v0_29_1.ts @@ -18,21 +18,16 @@ * D. Record — handled by the runner. */ -import { execSync } from 'child_process'; import type { BrainEngine } from '../../core/engine.ts'; import type { Migration, OrchestratorOpts, OrchestratorResult, OrchestratorPhaseResult } from './types.ts'; -import { childGlobalFlags } from '../../core/cli-options.ts'; // ── Phase A — Schema ──────────────────────────────────────── -function phaseASchema(opts: OrchestratorOpts): OrchestratorPhaseResult { +async function phaseASchema(opts: OrchestratorOpts): Promise { if (opts.dryRun) return { name: 'schema', status: 'skipped', detail: 'dry-run' }; try { - execSync('gbrain init --migrate-only' + childGlobalFlags(), { - stdio: 'inherit', - timeout: 600_000, // 10 min — duplicate-heavy installs can be slow - env: process.env, - }); + const { runMigrateOnlyCore } = await import('./in-process.ts'); + await runMigrateOnlyCore(); return { name: 'schema', status: 'complete' }; } catch (e) { return { name: 'schema', status: 'failed', detail: e instanceof Error ? e.message : String(e) }; @@ -129,7 +124,7 @@ async function orchestrator(opts: OrchestratorOpts): Promise const phases: OrchestratorPhaseResult[] = []; - const a = phaseASchema(opts); + const a = await phaseASchema(opts); phases.push(a); if (a.status === 'failed') return finalize(phases, 'failed'); diff --git a/src/commands/reindex.ts b/src/commands/reindex.ts index 7ace12c64..6e5245ac4 100644 --- a/src/commands/reindex.ts +++ b/src/commands/reindex.ts @@ -24,6 +24,7 @@ import type { BrainEngine } from '../core/engine.ts'; import { MARKDOWN_CHUNKER_VERSION } from '../core/chunkers/recursive.ts'; import { importFromContent, importFromFile } from '../core/import-file.ts'; +import { serializeMarkdown } from '../core/markdown.ts'; import { createProgress } from '../core/progress.ts'; import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts'; import { existsSync } from 'fs'; @@ -220,8 +221,24 @@ export async function runReindex(engine: BrainEngine, args: string[]): Promise }> } | undefined; try { + // v0.41.37.0 #1569: --no-schema-pack escape hatch. Skip pack load entirely so + // no user-supplied pack regex (markdown.ts subtype path_pattern) runs during + // sync; pages fall back to legacy prefix typing. + if (opts.noSchemaPack) { + serr('[sync] --no-schema-pack: skipping schema pack; pages use legacy prefix typing'); + throw new Error('schema-pack-skipped'); + } const { loadActivePack } = await import('../core/schema-pack/load-active.ts'); const { loadConfig } = await import('../core/config.ts'); const resolved = await loadActivePack({ @@ -1589,6 +1605,13 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise1 / --all the stuck file is the + // begin-line with no matching completion in the in-flight set. + if (process.env.GBRAIN_SYNC_TRACE) serr(`[sync] begin import: ${path}`); try { // v0.18.0+ multi-source: thread `opts.sourceId` so per-page tx writes // (putPage / getTags / addTag / removeTag / deleteChunks / upsertChunks @@ -2078,6 +2101,12 @@ Options: --watch Re-sync continuously on an interval. --interval N Watch-mode interval in seconds (default 60). --no-pull Skip 'git pull' before the sync (useful for tests). + --no-schema-pack Skip loading the active schema pack (no per-file pack + regex runs; pages use legacy prefix typing). Escape + hatch if a suspect pack regex is wedging sync. + PGLite is single-writer: stop 'gbrain serve' before a + large sync (see docs/architecture/serve-sync-concurrency.md). + GBRAIN_SYNC_TRACE=1 names the file being imported (hang triage). --all Sync every registered source instead of just the default (multi-source brains). --parallel N (with --all) Run up to N sources concurrently. @@ -2113,6 +2142,7 @@ See also: const noEmbed = args.includes('--no-embed'); const skipFailed = args.includes('--skip-failed'); const retryFailed = args.includes('--retry-failed'); + const noSchemaPack = args.includes('--no-schema-pack'); // v0.41.37.0 #1569 const syncAll = args.includes('--all'); const jsonOut = args.includes('--json'); const yesFlag = args.includes('--yes'); @@ -2540,7 +2570,7 @@ See also: repoPath: src.local_path!, dryRun, full, noPull, noEmbed: effectiveNoEmbed, - skipFailed, retryFailed, + skipFailed, retryFailed, noSchemaPack, sourceId: src.id, strategy: cfg.strategy, concurrency, @@ -2741,7 +2771,7 @@ See also: : undefined; singleSourceTimer?.unref?.(); const opts: SyncOpts = { - repoPath, dryRun, full, noPull, noEmbed, skipFailed, retryFailed, sourceId, + repoPath, dryRun, full, noPull, noEmbed, skipFailed, retryFailed, noSchemaPack, sourceId, strategy: strategyArg, concurrency, signal: singleSourceController?.signal, }; @@ -2901,6 +2931,8 @@ export async function syncOneSource( skipFailed: boolean; retryFailed: boolean; concurrency: number | undefined; + /** v0.41.37.0 #1569: propagate --no-schema-pack into every per-source sync. */ + noSchemaPack?: boolean; }, ): Promise<{ result: SyncResult; log: string }> { const cfg = (src.config || {}) as { strategy?: 'markdown' | 'code' | 'auto' }; @@ -2913,6 +2945,7 @@ export async function syncOneSource( noEmbed: shared.noEmbed, skipFailed: shared.skipFailed, retryFailed: shared.retryFailed, + noSchemaPack: shared.noSchemaPack, sourceId: src.id, strategy: cfg.strategy, concurrency: shared.concurrency, diff --git a/src/core/import-file.ts b/src/core/import-file.ts index e07996c93..e35709ba5 100644 --- a/src/core/import-file.ts +++ b/src/core/import-file.ts @@ -669,12 +669,25 @@ export async function importFromContent( ); } - // Tag reconciliation: remove stale, add current - const existingTags = await tx.getTags(slug, txOpts); - const newTags = new Set(parsed.tags); - for (const old of existingTags) { - if (!newTags.has(old)) await tx.removeTag(slug, old, txOpts); - } + // Tag reconciliation: ADD-ONLY (v0.41.37.0 #1621). + // + // We deliberately do NOT delete existing tags here. The `tags` table has + // no provenance column, and frontmatter tags are stripped from the stored + // `pages.frontmatter` (markdown.ts:118) — so at re-import time we cannot + // distinguish a frontmatter-origin tag from a DB-side enrichment tag + // (auto-tag / dream synthesize / signal-detector writes to the same + // table). The pre-v0.41.37.0 "delete every existing tag not in the current + // frontmatter" logic wiped ALL enrichment tags on every re-import — most + // visibly under `gbrain reindex --markdown` (#1621), which re-imports every + // page with forceRechunk. reindex is a re-chunk/re-embed op; it must not + // destroy tags. + // + // Trade-off (accepted): removing a tag from a page's frontmatter no longer + // removes it from the DB on the next sync. That staleness is minor (tags + // are additive metadata) and far preferable to silently losing enrichment + // tags. Frontmatter-tag REMOVAL would require a `tag_source` provenance + // column (deferred — see TODOS.md #1621-followup). addTag is idempotent + // (ON CONFLICT DO NOTHING), so re-adding existing tags is a no-op. for (const tag of parsed.tags) { await tx.addTag(slug, tag, txOpts); } diff --git a/src/core/schema-pack/link-inference.ts b/src/core/schema-pack/link-inference.ts index 59d2c0379..2682a612d 100644 --- a/src/core/schema-pack/link-inference.ts +++ b/src/core/schema-pack/link-inference.ts @@ -39,7 +39,7 @@ // universe. import type { SchemaPackManifest } from './manifest-v1.ts'; -import { PageRegexBudget } from './redos-guard.ts'; +import { PageRegexBudget, runRegexBounded } from './redos-guard.ts'; /** * Try to resolve a link verb from the active pack's declared @@ -80,12 +80,16 @@ export function inferLinkTypeFromPack( } if (match !== null) return lt.name; } else { - // No budget provided (test contexts) — run the regex directly. + // No budget provided (test contexts) — still route through the bounded + // executor so the v0.41.37.0 #1569 input-length cap + vm timeout apply. + // Previously this ran `new RegExp(pattern).test(context)` UNBOUNDED, the + // one ReDoS hole with no timeout. runRegexBounded throws on + // timeout/oversize/malformed → skip and continue (degrade to mentions). try { - if (new RegExp(pattern).test(context)) return lt.name; + if (runRegexBounded(pattern, context) !== null) return lt.name; } catch { - // Malformed pattern — skip and continue. Pack validation - // should have caught this at load. + // Timed out, oversize input, or malformed pattern — skip and continue. + // Pack validation + the star-height lint rule surface bad patterns. } } } diff --git a/src/core/schema-pack/lint-rules.ts b/src/core/schema-pack/lint-rules.ts index b0ebfcdce..131bcc743 100644 --- a/src/core/schema-pack/lint-rules.ts +++ b/src/core/schema-pack/lint-rules.ts @@ -320,6 +320,34 @@ export const mutationCountAnomaly: LintRule = (manifest, opts) => { // Aggregator // ──────────────────────────────────────────────────────────────────────── +// v0.41.37.0 #1569: advisory ReDoS pre-screen for pack inference regexes. +// Flags the classic nested-quantifier shapes ((a+)+, (a*)*, (a+)*, (\w+)+) +// that cause catastrophic backtracking. WARNING, not error: a hard reject +// would disable the whole pack on upgrade (pages fall back to legacy typing). +// The runtime input-length cap (MAX_REGEX_INPUT_CHARS in redos-guard.ts) is +// the actual safety net; this rule tells the author to fix the pattern. +// Heuristic: an inner group containing a +/* quantifier, wrapped by an outer +// +/* quantifier. Catches the common ReDoS class, not every possible one. +const NESTED_QUANTIFIER_RE = /\([^()]*[+*][^()]*\)\s*[+*]/; +export const linkRegexCatastrophicBacktrack: LintRule = (manifest) => { + const issues: LintIssue[] = []; + for (const lt of manifest.link_types) { + const pattern = lt.inference?.regex; + if (!pattern) continue; + if (NESTED_QUANTIFIER_RE.test(pattern)) { + issues.push({ + rule: 'link_regex_catastrophic_backtrack', + severity: 'warning', + message: `link_type '${lt.name}' inference.regex '${pattern}' has a nested quantifier (e.g. (a+)+) that can cause catastrophic backtracking (ReDoS)`, + pack: manifest.name, + link: lt.name, + hint: `rewrite without nested quantifiers (e.g. (a+)+ → a+). The runtime caps input length, but the pattern stays O(2^n) on adversarial input`, + }); + } + } + return issues; +}; + /** All rules. File-plane callers can compose a subset via FILE_PLANE_RULES. */ export const ALL_LINT_RULES: ReadonlyArray<{ name: string; rule: LintRule; planeAware: boolean }> = [ { name: 'alias_shadows_type', rule: aliasShadowsType, planeAware: false }, @@ -331,6 +359,7 @@ export const ALL_LINT_RULES: ReadonlyArray<{ name: string; rule: LintRule; plane { name: 'expert_routing_without_prefix', rule: expertRoutingWithoutPrefix, planeAware: false }, { name: 'prefix_collision', rule: prefixCollision, planeAware: false }, { name: 'prefix_strict_subset_overlap', rule: prefixStrictSubsetOverlap, planeAware: false }, + { name: 'link_regex_catastrophic_backtrack', rule: linkRegexCatastrophicBacktrack, planeAware: false }, { name: 'extractable_empty_corpus', rule: extractableEmptyCorpus, planeAware: true }, { name: 'mutation_count_anomaly', rule: mutationCountAnomaly, planeAware: true }, ]; diff --git a/src/core/schema-pack/redos-guard.ts b/src/core/schema-pack/redos-guard.ts index b0a2c444b..4153f62c4 100644 --- a/src/core/schema-pack/redos-guard.ts +++ b/src/core/schema-pack/redos-guard.ts @@ -37,6 +37,28 @@ import { runInContext, createContext } from 'node:vm'; export const LINK_EXTRACTION_TOTAL_BUDGET_MS = 500 as const; export const PER_REGEX_TIMEOUT_MS = 50 as const; +// v0.41.37.0 #1569: hard input-length cap. Catastrophic backtracking needs a +// long input to blow up; a pack regex run against a multi-KB body is the blast +// radius. Capping the input length removes it cheaply — a link-extraction +// `context` is normally a sentence or short paragraph, so 64KB is generous. +// Over the cap, the regex is skipped (degrade-to-mentions) without even +// entering the vm. This is the real runtime safety net (the star-height lint +// rule is advisory). Env-overridable for power users with huge contexts. +export const MAX_REGEX_INPUT_CHARS = (() => { + const raw = process.env.GBRAIN_MAX_REGEX_INPUT_CHARS; + const n = raw ? parseInt(raw, 10) : NaN; + return Number.isFinite(n) && n > 0 ? n : 64_000; +})(); + +/** Tagged error thrown when input exceeds MAX_REGEX_INPUT_CHARS. Treated as + * degrade-to-mentions by `PageRegexBudget.runBounded` (counts against budget). */ +export class RegexInputTooLargeError extends Error { + constructor(public readonly length: number) { + super(`regex input ${length} chars exceeds cap ${MAX_REGEX_INPUT_CHARS}`); + this.name = 'RegexInputTooLargeError'; + } +} + export class RegexTimeoutError extends Error { readonly verb: string; readonly pattern: string; @@ -123,6 +145,13 @@ export function runRegexBounded( text: string, timeoutMs: number = PER_REGEX_TIMEOUT_MS, ): RegExpMatchArray | null { + // v0.41.37.0 #1569: input-length cap BEFORE the vm. Over the cap, skip the + // regex entirely (the surrounding budget treats the throw as degrade). This + // is the primary ReDoS safety net — catastrophic backtracking can't blow up + // on input it never sees. + if (text.length > MAX_REGEX_INPUT_CHARS) { + throw new RegexInputTooLargeError(text.length); + } // Create a fresh context so the pack's regex can't leak state across // runs. Pass pattern + text as primitives only. const ctx = createContext({ pattern, text }); diff --git a/test/fix-wave-structural.test.ts b/test/fix-wave-structural.test.ts index af4d74151..985c9dad8 100644 --- a/test/fix-wave-structural.test.ts +++ b/test/fix-wave-structural.test.ts @@ -95,11 +95,16 @@ describe('v0.36.1.x #1077 — admin register-client supports PKCE public clients }); }); -describe('v0.36.1.x #1100 — PGLite v0.11.0 phaseASchema routes in-process', () => { - test('phaseASchema branches on pglite and calls initSchema directly', () => { +describe('v0.41.37.0 #1605 — v0.11.0 phaseASchema routes in-process for ALL engines', () => { + test('phaseASchema calls runMigrateOnlyCore (in-process) + is awaited', () => { + // Supersedes #1100's PGLite-only in-process branch. v0.41.37.0 #1605 routes + // EVERY engine through runMigrateOnlyCore (no execSync subprocess at all), + // which is strictly stronger: PGLite still never subprocesses, AND the + // Windows+Postgres getaddrinfo-ENOTFOUND spawn bug is closed too. + // The eng.initSchema() call moved into src/commands/migrations/in-process.ts. const src = readFileSync('src/commands/migrations/v0_11_0.ts', 'utf8'); - expect(src).toMatch(/cfg\?\.engine\s*===\s*'pglite'/); - expect(src).toMatch(/eng\.initSchema\(\)/); + expect(src).toContain('runMigrateOnlyCore()'); + expect(src).not.toContain("execSync('gbrain init --migrate-only'"); expect(src).toMatch(/await\s+phaseASchema/); }); diff --git a/test/import-file.test.ts b/test/import-file.test.ts index b0be5beb2..3e4231154 100644 --- a/test/import-file.test.ts +++ b/test/import-file.test.ts @@ -217,7 +217,11 @@ Same content. expect(putCall).toBeUndefined(); }); - test('reconciles tags: removes old, adds new', async () => { + test('reconciles tags: ADD-ONLY, never removes (v0.41.37.0 #1621)', async () => { + // Add-only reconciliation: re-import adds current frontmatter tags and + // NEVER deletes — so DB-side enrichment tags (here simulated as 'old-tag') + // survive. The pre-#1621 behavior deleted every existing tag not in the + // current frontmatter, wiping enrichment tags on every re-import/reindex. const filePath = join(TMP, 'retag.md'); writeFileSync(filePath, `--- type: concept @@ -239,9 +243,11 @@ Content here. const removeCalls = calls.filter((c: any) => c.method === 'removeTag'); const addCalls = calls.filter((c: any) => c.method === 'addTag'); - expect(removeCalls.length).toBe(1); - expect(removeCalls[0].args[1]).toBe('old-tag'); + // No removals — 'old-tag' (enrichment) is preserved. + expect(removeCalls.length).toBe(0); + // Both current frontmatter tags are added (idempotent via ON CONFLICT). expect(addCalls.length).toBe(2); + expect(addCalls.map((c: any) => c.args[1]).sort()).toEqual(['kept-tag', 'new-tag']); }); test('chunks compiled_truth and timeline separately', async () => { diff --git a/test/migration-in-process.serial.test.ts b/test/migration-in-process.serial.test.ts new file mode 100644 index 000000000..a460878d3 --- /dev/null +++ b/test/migration-in-process.serial.test.ts @@ -0,0 +1,112 @@ +// v0.41.37.0 #1605 — migration schema phases run IN-PROCESS (was a +// `gbrain init --migrate-only` subprocess that died with getaddrinfo ENOTFOUND +// on Windows+bun+Supabase). runMigrateOnlyCore is the single in-process path; +// runGbrainSubprocess captures child stderr for the remaining backfill spawns. +import { describe, test, expect } from 'bun:test'; +import { tmpdir } from 'os'; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, readdirSync } from 'fs'; +import { join } from 'path'; +import { withEnv } from './helpers/with-env.ts'; +import { PGLiteEngine } from '../src/core/pglite-engine.ts'; +import { + runMigrateOnlyCore, + runGbrainSubprocess, + MigrateOnlyError, +} from '../src/commands/migrations/in-process.ts'; + +const MIGRATION_DIR = join(import.meta.dir, '..', 'src', 'commands', 'migrations'); +const SCHEMA_PHASE_FILES = [ + 'v0_11_0', 'v0_12_0', 'v0_12_2', 'v0_13_0', 'v0_16_0', + 'v0_18_0', 'v0_18_1', 'v0_21_0', 'v0_29_1', +]; + +describe('#1605 runMigrateOnlyCore (in-process schema)', () => { + test('brings a fresh PGLite brain to head without spawning', async () => { + const home = mkdtempSync(join(tmpdir(), 'mip-')); + const dataDir = join(home, 'data'); + mkdirSync(join(home, '.gbrain'), { recursive: true }); + writeFileSync( + join(home, '.gbrain', 'config.json'), + JSON.stringify({ engine: 'pglite', database_path: dataDir }), + ); + + const result = await withEnv( + { GBRAIN_HOME: home, DATABASE_URL: undefined, GBRAIN_DATABASE_URL: undefined }, + () => runMigrateOnlyCore(), + ); + expect(result.engine).toBe('pglite'); + + // Verify schema landed: reconnect a fresh engine to the same data dir and + // confirm a core table exists (proves initSchema ran in-process). + const verify = new PGLiteEngine(); + await verify.connect({ database_path: dataDir }); + try { + const rows = await verify.executeRaw<{ t: string | null }>( + "SELECT to_regclass('public.pages')::text AS t", + ); + expect(rows[0]?.t).toBe('pages'); + } finally { + await verify.disconnect(); + } + }); + + test('throws MigrateOnlyError when no brain is configured', async () => { + const home = mkdtempSync(join(tmpdir(), 'mip-noconf-')); + await expect( + withEnv( + { GBRAIN_HOME: home, DATABASE_URL: undefined, GBRAIN_DATABASE_URL: undefined }, + () => runMigrateOnlyCore(), + ), + ).rejects.toBeInstanceOf(MigrateOnlyError); + }); +}); + +describe('#1605 runGbrainSubprocess (stderr capture)', () => { + test('folds child stderr into the thrown error', () => { + let msg = ''; + try { + runGbrainSubprocess("sh -c 'echo BOOM_STDERR 1>&2; exit 1'"); + } catch (e) { + msg = e instanceof Error ? e.message : String(e); + } + expect(msg).toContain('BOOM_STDERR'); + }); + + test('returns child stdout on success', () => { + const out = runGbrainSubprocess("sh -c 'echo hello-stdout'"); + expect(out).toContain('hello-stdout'); + }); +}); + +describe('#1605 structural guard: schema phases are in-process', () => { + test('no schema phase still execSyncs `gbrain init --migrate-only`', () => { + for (const f of SCHEMA_PHASE_FILES) { + const src = readFileSync(join(MIGRATION_DIR, `${f}.ts`), 'utf-8'); + expect(src).not.toContain("execSync('gbrain init --migrate-only'"); + } + }); + + test('every schema phase calls runMigrateOnlyCore + is awaited', () => { + for (const f of SCHEMA_PHASE_FILES) { + const src = readFileSync(join(MIGRATION_DIR, `${f}.ts`), 'utf-8'); + expect(src).toContain('runMigrateOnlyCore()'); + // phaseASchema must be async + awaited at its call site. + expect(src).toContain('async function phaseASchema'); + const awaited = src.includes('await phaseASchema(opts)') || + src.includes('push(await phaseASchema(opts))'); + expect(awaited).toBe(true); + } + }); + + test('NO migration orchestrator anywhere spawns `gbrain init --migrate-only`', () => { + // All-files invariant (not just the 9): the subprocess spawn is the + // Windows-ENOTFOUND bug class. Other files (v0_22_4, v0_28_0, v0_31_0, + // v0_14_0, v0_32_2) define an in-process phaseASchema that never spawned — + // those are fine. We only ban the spawn literal. + const files = readdirSync(MIGRATION_DIR).filter(n => /^v\d/.test(n) && n.endsWith('.ts')); + for (const n of files) { + const src = readFileSync(join(MIGRATION_DIR, n), 'utf-8'); + expect(src).not.toContain("execSync('gbrain init --migrate-only'"); + } + }); +}); diff --git a/test/migrations-v0_13_0.test.ts b/test/migrations-v0_13_0.test.ts index 6de75bcb5..6882dd505 100644 --- a/test/migrations-v0_13_0.test.ts +++ b/test/migrations-v0_13_0.test.ts @@ -59,12 +59,16 @@ describe('v0.13.0 — Frontmatter relationship indexing migration', () => { expect(src).not.toMatch(/\$\{GBRAIN\}/); }); - test('phase commands invoke bare `gbrain` shell-out (Bug 1 fix)', () => { + test('phases use in-process schema + bare `gbrain` subprocess (v0.41.37.0 #1605)', () => { const src = readFileSync(SRC_PATH, 'utf-8'); - // All three phases shell out to bare `gbrain` so the canonical shim - // on PATH wins. This is the shape v0_12_0 has always used. - expect(src).toContain("execSync('gbrain init --migrate-only'"); - expect(src).toContain("execSync('gbrain extract links --source db --include-frontmatter'"); + // Schema bring-up is now IN-PROCESS (was execSync('gbrain init + // --migrate-only'), which died with getaddrinfo ENOTFOUND on Windows). + expect(src).toContain('runMigrateOnlyCore()'); + expect(src).not.toContain("execSync('gbrain init --migrate-only'"); + // Backfill extract goes through the stderr-capturing wrapper (still bare + // `gbrain` so the canonical shim on PATH wins). + expect(src).toContain("runGbrainSubprocess('gbrain extract links --source db --include-frontmatter'"); + // Stats readback still shells out (reads stdout); bare gbrain. expect(src).toContain("execSync('gbrain call get_stats'"); }); diff --git a/test/migrations-v0_13_1-grandfather.test.ts b/test/migrations-v0_13_1-grandfather.test.ts new file mode 100644 index 000000000..82249c497 --- /dev/null +++ b/test/migrations-v0_13_1-grandfather.test.ts @@ -0,0 +1,144 @@ +// v0.41.37.0 #1581 — chunked, source-safe, soft-delete-filtered grandfather pass. +// +// The pre-#1581 per-page getPage+putPage loop hung CPU-bound for 70+ min on an +// 82K-page PGLite brain. The rewrite is a chunked bulk SQL pass keyed on +// pages.id (NOT slug — slug isn't globally unique), filtering deleted_at IS NULL. +// These tests drive phaseCGrandfather directly against a real PGLite engine. +import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test'; +import { tmpdir } from 'os'; +import { mkdtempSync, existsSync, readFileSync } from 'fs'; +import { join } from 'path'; +import { PGLiteEngine } from '../src/core/pglite-engine.ts'; +import { resetPgliteState } from './helpers/reset-pglite.ts'; +import { withEnv } from './helpers/with-env.ts'; +import { phaseCGrandfather } from '../src/commands/migrations/v0_13_1.ts'; +import type { OrchestratorOpts } from '../src/commands/migrations/types.ts'; + +let engine: PGLiteEngine; + +beforeAll(async () => { + engine = new PGLiteEngine(); + await engine.connect({}); + await engine.initSchema(); +}); + +afterAll(async () => { + await engine.disconnect(); +}); + +beforeEach(async () => { + await resetPgliteState(engine); +}); + +const ensureSource = (id: string) => + engine.executeRaw('INSERT INTO sources (id, name) VALUES ($1, $1) ON CONFLICT DO NOTHING', [id]); + +const seed = (slug: string, frontmatter: Record, sourceId = 'default') => + engine.putPage(slug, { + type: 'concept', title: slug, compiled_truth: 'body', timeline: '', frontmatter, + }, { sourceId }); + +const fmOf = async (slug: string, sourceId = 'default') => { + const p = await engine.getPage(slug, { sourceId }); + return p?.frontmatter ?? null; +}; + +// Run the phase with an isolated GBRAIN_HOME so the rollback log lands in a +// tempdir (withEnv keeps R1 test-isolation compliance — no raw process.env writes). +async function gf(home: string, opts: Partial = {}) { + const full: OrchestratorOpts = { yes: true, dryRun: false, noAutopilotInstall: true, ...opts }; + return withEnv({ GBRAIN_HOME: home }, () => phaseCGrandfather(engine, full)); +} + +describe('#1581 phaseCGrandfather (chunked, source-safe)', () => { + test('absent validate key → validate:false', async () => { + const home = mkdtempSync(join(tmpdir(), 'gf-')); + await seed('a', { foo: 1 }); + const { result } = await gf(home); + expect(result.status).toBe('complete'); + expect((await fmOf('a'))?.validate).toBe(false); + }); + + test('explicit validate (true/false/null) is NOT modified', async () => { + const home = mkdtempSync(join(tmpdir(), 'gf-')); + await seed('t', { validate: true }); + await seed('f', { validate: false }); + await seed('n', { validate: null }); + await gf(home); + expect((await fmOf('t'))?.validate).toBe(true); + expect((await fmOf('f'))?.validate).toBe(false); + // null value still counts as "key present" → left untouched. + expect((await fmOf('n'))?.validate ?? 'WAS_NULL').toBe('WAS_NULL'); + }); + + test('soft-deleted pages are NOT grandfathered', async () => { + const home = mkdtempSync(join(tmpdir(), 'gf-')); + await seed('live', { foo: 1 }); + await seed('dead', { foo: 1 }); + await engine.softDeletePage('dead'); + await gf(home); + + expect((await fmOf('live'))?.validate).toBe(false); + const rows = await engine.executeRaw<{ fm: Record }>( + "SELECT frontmatter AS fm FROM pages WHERE slug = 'dead'", + ); + expect(Object.prototype.hasOwnProperty.call(rows[0]?.fm ?? {}, 'validate')).toBe(false); + }); + + test('multi-source duplicate slugs: both grandfathered, no cross-contamination', async () => { + const home = mkdtempSync(join(tmpdir(), 'gf-')); + await ensureSource('src2'); + await seed('people/dup', { src: 'a' }, 'default'); + await seed('people/dup', { src: 'b' }, 'src2'); + await gf(home); + + expect((await fmOf('people/dup', 'default'))?.validate).toBe(false); + expect((await fmOf('people/dup', 'src2'))?.validate).toBe(false); + expect((await fmOf('people/dup', 'default'))?.src).toBe('a'); + expect((await fmOf('people/dup', 'src2'))?.src).toBe('b'); + }); + + test('rollback log carries source identity', async () => { + const home = mkdtempSync(join(tmpdir(), 'gf-')); + await ensureSource('src2'); + await seed('people/dup', { src: 'a' }, 'default'); + await seed('people/dup', { src: 'b' }, 'src2'); + await gf(home); + + // configDir() appends '.gbrain' to GBRAIN_HOME. + const logPath = join(home, '.gbrain', 'migrations', 'v0_13_1-rollback.jsonl'); + expect(existsSync(logPath)).toBe(true); + const lines = readFileSync(logPath, 'utf-8').trim().split('\n').map(l => JSON.parse(l)); + expect(lines.map(l => l.source_id).sort()).toEqual(['default', 'src2']); + expect(lines.every(l => typeof l.id === 'number' && l.slug === 'people/dup')).toBe(true); + }); + + test('re-run is a no-op (idempotent)', async () => { + const home = mkdtempSync(join(tmpdir(), 'gf-')); + await seed('a', { foo: 1 }); + const first = await gf(home); + expect(first.detail.touched).toBe(1); + const second = await gf(home); + expect(second.detail.touched).toBe(0); + }); + + test('dry-run counts without mutating', async () => { + const home = mkdtempSync(join(tmpdir(), 'gf-')); + await seed('a', { foo: 1 }); + const { detail } = await gf(home, { dryRun: true }); + expect(detail.touched).toBe(1); + expect((await fmOf('a'))?.validate ?? 'UNSET').toBe('UNSET'); + }); + + test('large-N (1200 pages) completes via chunked pass (hang regression)', async () => { + const home = mkdtempSync(join(tmpdir(), 'gf-')); + for (let i = 0; i < 1200; i++) await seed(`bulk/${i}`, { i }); + const t0 = Date.now(); + const { result, detail } = await gf(home); + expect(result.status).toBe('complete'); + expect(detail.touched).toBe(1200); + expect((await fmOf('bulk/0'))?.validate).toBe(false); + expect((await fmOf('bulk/1199'))?.validate).toBe(false); + expect(Date.now() - t0).toBeLessThan(30_000); + }); +}); diff --git a/test/migrations-v0_16_0.test.ts b/test/migrations-v0_16_0.test.ts index 2f12592d3..ffa32f74b 100644 --- a/test/migrations-v0_16_0.test.ts +++ b/test/migrations-v0_16_0.test.ts @@ -54,8 +54,9 @@ describe('v0.16.0 migration', () => { ]); }); - test('phaseASchema skips on dry-run', () => { - const r = __testing.phaseASchema({ dryRun: true, yes: true, noAutopilotInstall: true }); + test('phaseASchema skips on dry-run', async () => { + // v0.41.37.0 #1605: phaseASchema is now async (in-process runMigrateOnlyCore). + const r = await __testing.phaseASchema({ dryRun: true, yes: true, noAutopilotInstall: true }); expect(r.status).toBe('skipped'); expect(r.detail).toBe('dry-run'); }); diff --git a/test/redos-hardening.test.ts b/test/redos-hardening.test.ts new file mode 100644 index 000000000..e122207ee --- /dev/null +++ b/test/redos-hardening.test.ts @@ -0,0 +1,96 @@ +// v0.41.37.0 #1569 — ReDoS hardening + diagnostics for schema-pack regexes. +// +// The real exposure was link-inference.ts:85 running `new RegExp(pattern) +// .test(context)` UNBOUNDED when no PageRegexBudget was passed. Fix: an input- +// length cap (the runtime safety net) + routing that path through the bounded +// executor + an advisory star-height lint rule. +import { describe, test, expect } from 'bun:test'; +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { + runRegexBounded, + PageRegexBudget, + RegexInputTooLargeError, + MAX_REGEX_INPUT_CHARS, +} from '../src/core/schema-pack/redos-guard.ts'; +import { inferLinkTypeFromPack } from '../src/core/schema-pack/link-inference.ts'; +import { linkRegexCatastrophicBacktrack } from '../src/core/schema-pack/lint-rules.ts'; +import type { SchemaPackManifest } from '../src/core/schema-pack/manifest-v1.ts'; + +describe('#1569 input-length cap', () => { + test('runRegexBounded throws RegexInputTooLargeError over the cap', () => { + const big = 'a'.repeat(MAX_REGEX_INPUT_CHARS + 1); + expect(() => runRegexBounded('a', big)).toThrow(RegexInputTooLargeError); + }); + + test('runRegexBounded works normally under the cap', () => { + const m = runRegexBounded('wor', 'hello world'); + expect(m).not.toBeNull(); + expect(runRegexBounded('zzz', 'hello world')).toBeNull(); + }); + + test('PageRegexBudget.runBounded degrades (null) on oversize input', () => { + const budget = new PageRegexBudget(); + const big = 'a'.repeat(MAX_REGEX_INPUT_CHARS + 1); + expect(budget.runBounded('verb', 'a', big)).toBeNull(); + }); + + test('inferLinkTypeFromPack (no budget) does NOT run regex unbounded on huge input', () => { + // Pre-#1569 this ran new RegExp().test() with no length cap → ReDoS risk. + // A catastrophic pattern + a long input must NOT hang; the cap skips it. + const pack = { + link_types: [{ name: 'founded', inference: { regex: '(a+)+$' } }], + } as unknown as Pick; + const huge = 'a'.repeat(MAX_REGEX_INPUT_CHARS + 100) + '!'; + const t0 = Date.now(); + const result = inferLinkTypeFromPack(pack, 'company', huge); + // Skipped via the cap → no match, and fast (no catastrophic backtrack). + expect(result).toBeNull(); + expect(Date.now() - t0).toBeLessThan(2_000); + }); +}); + +describe('#1569 star-height lint rule', () => { + const mk = (regex: string): SchemaPackManifest => + ({ name: 'testpack', page_types: [], link_types: [{ name: 'founded', inference: { regex } }] }) as unknown as SchemaPackManifest; + + test('flags classic nested-quantifier shapes as warnings', () => { + for (const bad of ['(a+)+', '(a*)*', '(a+)*', '(\\w+)+$', '(.*)+']) { + const issues = linkRegexCatastrophicBacktrack(mk(bad)) as ReturnType & any[]; + expect(issues.length).toBe(1); + expect(issues[0].severity).toBe('warning'); + expect(issues[0].rule).toBe('link_regex_catastrophic_backtrack'); + expect(issues[0].link).toBe('founded'); + } + }); + + test('does NOT flag benign patterns', () => { + for (const ok of ['a+', '(abc)+', '(a|b)+', 'founded\\s+\\w+', '[a-z]+@[a-z]+']) { + const issues = linkRegexCatastrophicBacktrack(mk(ok)) as any[]; + expect(issues.length).toBe(0); + } + }); + + test('no regex → no issue', () => { + const manifest = { name: 'p', page_types: [], link_types: [{ name: 'x' }] } as unknown as SchemaPackManifest; + expect((linkRegexCatastrophicBacktrack(manifest) as any[]).length).toBe(0); + }); +}); + +describe('#1569 --no-schema-pack + heartbeat wiring (structural)', () => { + const SYNC = readFileSync(join(import.meta.dir, '..', 'src', 'commands', 'sync.ts'), 'utf-8'); + + test('SyncOpts carries noSchemaPack and it gates loadActivePack', () => { + expect(SYNC).toContain('noSchemaPack?: boolean'); + expect(SYNC).toContain("args.includes('--no-schema-pack')"); + expect(SYNC).toContain('if (opts.noSchemaPack)'); + }); + + test('begin heartbeat fires before importFile (GBRAIN_SYNC_TRACE)', () => { + const beginIdx = SYNC.indexOf('begin import:'); + const importIdx = SYNC.indexOf('importFile(eng, filePath, path'); + expect(beginIdx).toBeGreaterThan(0); + expect(importIdx).toBeGreaterThan(0); + expect(beginIdx).toBeLessThan(importIdx); + }); +}); diff --git a/test/reindex-preserve-tags.test.ts b/test/reindex-preserve-tags.test.ts new file mode 100644 index 000000000..762195859 --- /dev/null +++ b/test/reindex-preserve-tags.test.ts @@ -0,0 +1,75 @@ +// v0.41.37.0 #1621 — reindex / re-import must NOT wipe DB-side enrichment tags. +// +// Root cause: import-file.ts reconciled tags from frontmatter only via +// DELETE-then-INSERT. The tags table has no provenance column and frontmatter +// tags are stripped from stored frontmatter (markdown.ts:118), so every +// re-import (notably `gbrain reindex --markdown`, which re-imports with +// forceRechunk) deleted all enrichment / dream / signal-detector tags. +// +// Fix: ADD-ONLY reconciliation. Re-import adds current frontmatter tags and +// never deletes. Accepted trade-off: removing a frontmatter tag no longer +// removes it from the DB (additive metadata; far better than wiping enrichment). +import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test'; +import { PGLiteEngine } from '../src/core/pglite-engine.ts'; +import { resetPgliteState } from './helpers/reset-pglite.ts'; +import { importFromContent } from '../src/core/import-file.ts'; + +let engine: PGLiteEngine; + +beforeAll(async () => { + engine = new PGLiteEngine(); + await engine.connect({}); + await engine.initSchema(); +}); + +afterAll(async () => { + await engine.disconnect(); +}); + +beforeEach(async () => { + await resetPgliteState(engine); +}); + +const page = (tags: string[]) => + `---\ntype: concept\ntitle: Xavi\ntags: [${tags.join(', ')}]\n---\nBody text for the page.\n`; + +describe('#1621 add-only tag reconciliation', () => { + test('re-import (reindex) preserves DB-side enrichment tags', async () => { + await importFromContent(engine, 'people/xavi', page(['founder', 'yc']), { forceRechunk: true, noEmbed: true }); + // Simulate DB-side enrichment writing a tag not present in frontmatter. + await engine.addTag('people/xavi', 'enrichment-tag'); + + // Re-import the same content (what reindex --markdown does). + await importFromContent(engine, 'people/xavi', page(['founder', 'yc']), { forceRechunk: true, noEmbed: true }); + + const tags = await engine.getTags('people/xavi'); + expect(tags.sort()).toEqual(['enrichment-tag', 'founder', 'yc']); + }); + + test('frontmatter tags are still added on import', async () => { + await importFromContent(engine, 'people/xavi', page(['founder', 'yc']), { forceRechunk: true, noEmbed: true }); + const tags = await engine.getTags('people/xavi'); + expect(tags.sort()).toEqual(['founder', 'yc']); + }); + + test('add-only: removing a frontmatter tag does NOT remove it (accepted trade-off)', async () => { + await importFromContent(engine, 'people/xavi', page(['founder', 'yc']), { forceRechunk: true, noEmbed: true }); + await engine.addTag('people/xavi', 'enrichment-tag'); + + // User removes "yc" from frontmatter and re-imports. + await importFromContent(engine, 'people/xavi', page(['founder']), { forceRechunk: true, noEmbed: true }); + + const tags = await engine.getTags('people/xavi'); + // "yc" lingers (add-only); enrichment-tag preserved; founder present. + expect(tags.sort()).toEqual(['enrichment-tag', 'founder', 'yc']); + }); + + test('adding a new frontmatter tag on re-import works (idempotent add)', async () => { + await importFromContent(engine, 'people/xavi', page(['founder']), { forceRechunk: true, noEmbed: true }); + await engine.addTag('people/xavi', 'enrichment-tag'); + await importFromContent(engine, 'people/xavi', page(['founder', 'growth']), { forceRechunk: true, noEmbed: true }); + + const tags = await engine.getTags('people/xavi'); + expect(tags.sort()).toEqual(['enrichment-tag', 'founder', 'growth']); + }); +}); diff --git a/test/schema-pack-lint-rules.test.ts b/test/schema-pack-lint-rules.test.ts index ca443ac3c..7fb08db90 100644 --- a/test/schema-pack-lint-rules.test.ts +++ b/test/schema-pack-lint-rules.test.ts @@ -332,12 +332,13 @@ describe('runAllLintRules — composition', () => { }); describe('rule registry shape', () => { - it('ALL_LINT_RULES contains 11 rules', () => { - expect(ALL_LINT_RULES.length).toBe(11); + it('ALL_LINT_RULES contains 12 rules', () => { + // v0.41.37.0 #1569 added link_regex_catastrophic_backtrack (file-plane). + expect(ALL_LINT_RULES.length).toBe(12); }); it('FILE_PLANE_LINT_RULES excludes the 2 DB-aware rules', () => { - expect(FILE_PLANE_LINT_RULES.length).toBe(9); + expect(FILE_PLANE_LINT_RULES.length).toBe(10); expect(FILE_PLANE_LINT_RULES.every((r) => !r.planeAware)).toBe(true); });