From 02ba4b4fc2cc4fea9729fe124250faa7056f2f9e Mon Sep 17 00:00:00 2001 From: Michael Gandal Date: Wed, 22 Jul 2026 15:38:30 -0400 Subject: [PATCH 01/25] dims: thread Matryoshka dimensions for Qwen3-Embedding on Ollama (#1072) Qwen3-Embedding family on Ollama supports Matryoshka truncation via the 'dimensions' field on /v1/embeddings. Without this passthrough, gbrain ignores user-selected reduced dims and the provider returns its native size, causing dim-mismatch errors against brains configured for narrower widths (e.g. existing 1536-dim brains). Matches by bare name 'qwen3-embedding' or any tag variant 'qwen3-embedding:0.6b' / ':4b' / ':8b'. Native dims: 0.6B=1024, 4B=2560, 8B=4096. All MRL-truncatable. 5 new tests; full AI suite 137/137 green. Co-authored-by: Claude Opus 4.7 (1M context) Co-authored-by: Time Attakc <89218912+time-attack@users.noreply.github.com> --- src/core/ai/dims.ts | 9 +++++++ test/ai/recipe-ollama-dims.test.ts | 41 ++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 test/ai/recipe-ollama-dims.test.ts diff --git a/src/core/ai/dims.ts b/src/core/ai/dims.ts index 7961516a5..b88a4e175 100644 --- a/src/core/ai/dims.ts +++ b/src/core/ai/dims.ts @@ -263,6 +263,15 @@ export function dimsProviderOptions( if (modelId === 'text-embedding-v3' || modelId === 'embedding-3') { return { openaiCompatible: { dimensions: dims } }; } + // Qwen3-Embedding family on Ollama (and any other openai-compatible + // provider serving it) supports Matryoshka truncation via `dimensions`. + // Native sizes: 0.6B=1024, 4B=2560, 8B=4096. Without `dimensions`, + // Ollama returns the native size and brains configured for narrower + // widths hard-fail with a dim-mismatch error. Pattern match the bare + // model name + any `:tag` (e.g. `qwen3-embedding:4b`, `qwen3-embedding:0.6b`). + if (modelId === 'qwen3-embedding' || modelId.startsWith('qwen3-embedding:')) { + return { openaiCompatible: { dimensions: dims } }; + } // MiniMax embo-01 takes a `type: 'db' | 'query'` field for asymmetric // retrieval. Today still hardcoded to 'db' for back-compat — opting // into the new inputType seam is a follow-up (see plan's deferred diff --git a/test/ai/recipe-ollama-dims.test.ts b/test/ai/recipe-ollama-dims.test.ts new file mode 100644 index 000000000..4f1d86ce8 --- /dev/null +++ b/test/ai/recipe-ollama-dims.test.ts @@ -0,0 +1,41 @@ +/** + * Ollama Matryoshka dims passthrough. + * + * Several embedding models served via Ollama (Qwen3-Embedding family) support + * Matryoshka truncation through the `dimensions` field on /v1/embeddings. + * Without this passthrough, gbrain ignores user-selected reduced dims and the + * provider returns its native size, causing dim-mismatch failures against + * brains configured for smaller widths. + */ + +import { describe, expect, test } from 'bun:test'; +import { dimsProviderOptions } from '../../src/core/ai/dims.ts'; + +describe('dims: ollama Matryoshka models', () => { + test('qwen3-embedding:4b threads dimensions=1536', () => { + expect(dimsProviderOptions('openai-compatible', 'qwen3-embedding:4b', 1536)) + .toEqual({ openaiCompatible: { dimensions: 1536 } }); + }); + + test('qwen3-embedding:0.6b threads dimensions=512', () => { + expect(dimsProviderOptions('openai-compatible', 'qwen3-embedding:0.6b', 512)) + .toEqual({ openaiCompatible: { dimensions: 512 } }); + }); + + test('qwen3-embedding:8b threads dimensions=2048', () => { + expect(dimsProviderOptions('openai-compatible', 'qwen3-embedding:8b', 2048)) + .toEqual({ openaiCompatible: { dimensions: 2048 } }); + }); + + test('bare qwen3-embedding (no quant tag) also recognized', () => { + expect(dimsProviderOptions('openai-compatible', 'qwen3-embedding', 1024)) + .toEqual({ openaiCompatible: { dimensions: 1024 } }); + }); + + test('unrelated openai-compat model returns undefined (regression guard)', () => { + expect(dimsProviderOptions('openai-compatible', 'nomic-embed-text', 768)) + .toBeUndefined(); + expect(dimsProviderOptions('openai-compatible', 'mxbai-embed-large', 1024)) + .toBeUndefined(); + }); +}); From d69f211629e2fcdb24ab6fb6105982f2ea3830e8 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Wed, 22 Jul 2026 13:41:32 -0700 Subject: [PATCH 02/25] fix(ci): delta-assert reporter leak test + raise shard timeout to 22min (#3231) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The signal-handler test asserted an absolute liveReporters===0 on a module-global set, so any other test file in the shard holding a live reporter flaked it — it red-flagged ~12 unrelated PR runs and one master push in two days, purely as a function of shard composition. The delta form pins the same claim (50 reporter lifecycles leak nothing). The 15-minute shard timeout cancelled 13 fully-passing runs under parallel PR load (PGLite WASM cold-starts stretch shards); the test-status gate then reported the cancellations as failures. Co-authored-by: Garry Tan Co-authored-by: Claude Fable 5 --- .github/workflows/test.yml | 6 +++++- test/progress.test.ts | 10 ++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6a80e4b77..db63724f8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -206,7 +206,11 @@ jobs: needs: cache-check if: needs.cache-check.outputs.hit != 'true' runs-on: ubuntu-latest - timeout-minutes: 15 + # 22, not 15: under parallel PR load the PGLite WASM cold-starts stretch a + # shard past 15 min while every test is still passing — the timeout then + # cancels the job and the test-status gate reads it as a failure. 13 runs + # died this way on 2026-07-21/22 alone. + timeout-minutes: 22 strategy: fail-fast: false matrix: diff --git a/test/progress.test.ts b/test/progress.test.ts index 0e70e0d6f..7f266d770 100644 --- a/test/progress.test.ts +++ b/test/progress.test.ts @@ -218,15 +218,21 @@ describe('progress reporter', () => { test('only one process-level signal handler installed across many reporters', () => { // Baseline: one handler already installed by prior tests in this file. const installedBefore = __signalHandlerInstalledForTest(); + // liveReporters is module-global, so a reporter left running by ANOTHER + // test file in the same shard shows up here. Assert the DELTA (these 50 + // lifecycles leak nothing) instead of an absolute zero — the absolute + // form flaked whenever shard composition changed and an unrelated file + // held a live reporter across this test. + const liveBefore = __liveReporterCountForTest(); const { stream } = sink(false); for (let i = 0; i < 50; i++) { const p = createProgress({ mode: 'json', stream, minIntervalMs: 0, minItems: 1 }); p.start(`phase_${i}`, 1); p.finish(); } - // After 50 reporter lifecycles, still exactly one handler and zero leaked live entries. + // After 50 reporter lifecycles, still exactly one handler and no new live entries. expect(__signalHandlerInstalledForTest()).toBe(installedBefore || true); - expect(__liveReporterCountForTest()).toBe(0); + expect(__liveReporterCountForTest()).toBe(liveBefore); }); test('startHeartbeat() fires heartbeats and stop() clears', async () => { From 2840734d702da6a594f7594b4580c222fb593d0e Mon Sep 17 00:00:00 2001 From: samporter-31 <88817805+samporter-31@users.noreply.github.com> Date: Thu, 23 Jul 2026 06:43:54 +0930 Subject: [PATCH 03/25] fix(doctor): normalize CRLF in extractTriggers so Windows skill triggers parse (#1149) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On Windows, `core.autocrlf=true` is the default and SKILL.md files are checked out with CRLF line endings. `extractTriggers` used regexes anchored to `\n` (`/^---\n.../` and `/^triggers:\s*\n.../`), which never matched `\r\n`, so the parser returned `[]` for every skill. Result: `gbrain doctor --fast --json` on Windows reported every skill not in `OVERLAP_WHITELIST` (39 of 42) as a false `mece_gap` warning — even though `skill_conformance` in the same run reported "42/42 skills pass". CI runs Ubuntu-only so the divergence never surfaced. Fix: normalize CRLF → LF at the top of `extractTriggers`. Single-line change preserves existing LF behavior. Function is now exported so the test can target it directly. Tests: added `describe("extractTriggers")` block covering LF input, CRLF input (regression case), missing frontmatter, missing triggers field, and quote-stripping. All 30 tests in `check-resolvable.test.ts` pass. Verified locally on Windows: `gbrain doctor --fast --json` now reports `resolver_health: ok, 42 skills, all reachable` (health_score 90 → 95). Co-authored-by: Claude Opus 4.7 (1M context) Co-authored-by: Time Attakc <89218912+time-attack@users.noreply.github.com> --- src/core/check-resolvable.ts | 16 +++++++++++++--- test/check-resolvable.test.ts | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/src/core/check-resolvable.ts b/src/core/check-resolvable.ts index a82b4a2c1..9f2f72980 100644 --- a/src/core/check-resolvable.ts +++ b/src/core/check-resolvable.ts @@ -217,9 +217,19 @@ export function parseResolverEntries(resolverContent: string): ResolverEntry[] { // `skillsDir/*/SKILL.md` when manifest.json is missing — the scenario // needed for AGENTS.md-only OpenClaw deployments. See D-CX-12 / F-ENG-1. -/** Simple YAML frontmatter parser — extracts triggers array if present. */ -function extractTriggers(skillContent: string): string[] { - const fmMatch = skillContent.match(/^---\n([\s\S]*?)\n---/); +/** + * Simple YAML frontmatter parser — extracts triggers array if present. + * + * Normalizes CRLF → LF before parsing so Windows checkouts (where + * `core.autocrlf=true` is the default) parse correctly. Without this, + * the `^---\n` and `^triggers:\s*\n` regexes never match because the + * file content is `---\r\n` / `triggers:\r\n`, and every skill on + * Windows is reported as `mece_gap` regardless of its actual content. + * CI runs on Ubuntu-only so the bug only surfaces in user environments. + */ +export function extractTriggers(skillContent: string): string[] { + const content = skillContent.replace(/\r\n/g, '\n'); + const fmMatch = content.match(/^---\n([\s\S]*?)\n---/); if (!fmMatch) return []; const fm = fmMatch[1]; const triggersMatch = fm.match(/^triggers:\s*\n((?:\s+-\s+.+\n?)*)/m); diff --git a/test/check-resolvable.test.ts b/test/check-resolvable.test.ts index 7e0c10e1a..b26040b85 100644 --- a/test/check-resolvable.test.ts +++ b/test/check-resolvable.test.ts @@ -6,6 +6,7 @@ import { checkResolvable, parseResolverEntries, extractDelegationTargets, + extractTriggers, } from "../src/core/check-resolvable.ts"; const SKILLS_DIR = join(import.meta.dir, "..", "skills"); @@ -195,6 +196,39 @@ describe("parseResolverEntries", () => { }); }); +describe("extractTriggers", () => { + const LF_FRONTMATTER = + "---\nname: query\ndescription: Test\ntriggers:\n - \"what do we know\"\n - \"tell me about\"\ntools:\n - search\n---\n\n# Body\n"; + + test("parses triggers from LF-terminated frontmatter", () => { + const triggers = extractTriggers(LF_FRONTMATTER); + expect(triggers).toEqual(["what do we know", "tell me about"]); + }); + + test("parses triggers from CRLF-terminated frontmatter (Windows checkouts)", () => { + // Regression: `core.autocrlf=true` is the Windows default. Without + // CRLF→LF normalization, every Windows skill is reported as a false + // mece_gap warning because the `^---\n` regex never matches `---\r\n`. + const crlf = LF_FRONTMATTER.replace(/\n/g, "\r\n"); + const triggers = extractTriggers(crlf); + expect(triggers).toEqual(["what do we know", "tell me about"]); + }); + + test("returns [] when frontmatter is missing", () => { + expect(extractTriggers("# Just a body, no frontmatter\n")).toEqual([]); + }); + + test("returns [] when triggers field is absent from frontmatter", () => { + const fm = "---\nname: query\ndescription: Test\ntools:\n - search\n---\n"; + expect(extractTriggers(fm)).toEqual([]); + }); + + test("strips surrounding quotes from trigger values", () => { + const fm = "---\nname: x\ntriggers:\n - \"double quoted\"\n - 'single quoted'\n - unquoted\n---\n"; + expect(extractTriggers(fm)).toEqual(["double quoted", "single quoted", "unquoted"]); + }); +}); + describe("checkResolvable — real skills directory", () => { const report = checkResolvable(SKILLS_DIR); From e78ad9ff9e2e6663c772847e432cad223d9a8b84 Mon Sep 17 00:00:00 2001 From: Richard Baker Date: Wed, 22 Jul 2026 17:47:11 -0400 Subject: [PATCH 04/25] fix(salience): exclude briefings/* from their own Brain Pulse (TIM-37) (#1202) The cron daily briefing writes 90_Briefings/.md, which gets re-ingested on the next sync and then dominates tomorrow's getRecentSalience output as pure self-reference (observed: top result score 0.9956, everyone else clustered at 0.587). Filter `p.slug LIKE 'briefings/%'` out of getRecentSalience in both the PG and PGLite engines. Suppressed by default; callers can still opt in by passing `slugPrefix: 'briefings/'` (or `--kind briefings/` from the CLI). search and list_pages are unaffected. Co-authored-by: CTO Co-authored-by: Paperclip Co-authored-by: Time Attakc <89218912+time-attack@users.noreply.github.com> --- src/core/pglite-engine.ts | 6 ++++++ src/core/postgres-engine.ts | 8 ++++++++ test/e2e/salience-pglite.test.ts | 23 +++++++++++++++++++++++ 3 files changed, 37 insertions(+) diff --git a/src/core/pglite-engine.ts b/src/core/pglite-engine.ts index 257090a23..e11eeb3fb 100644 --- a/src/core/pglite-engine.ts +++ b/src/core/pglite-engine.ts @@ -5837,6 +5837,11 @@ export class PGLiteEngine implements BrainEngine { params.push(escaped); prefixCondition = `AND p.slug LIKE $${params.length} ESCAPE '\\'`; } + // TIM-37: exclude briefing pages from their own Brain Pulse. See the + // matching block in postgres-engine.ts getRecentSalience() for context. + const excludeBriefings = !(slugPrefix && slugPrefix.startsWith('briefings')) + ? `AND p.slug NOT LIKE 'briefings/%'` + : ''; params.push(limit); const limitParam = `$${params.length}`; @@ -5872,6 +5877,7 @@ export class PGLiteEngine implements BrainEngine { LEFT JOIN takes t ON t.page_id = p.id AND t.active = TRUE WHERE GREATEST(p.updated_at, COALESCE(p.salience_touched_at, p.updated_at)) >= $1::timestamptz ${prefixCondition} + ${excludeBriefings} GROUP BY p.id ORDER BY score DESC LIMIT ${limitParam}`, diff --git a/src/core/postgres-engine.ts b/src/core/postgres-engine.ts index 0e8f658f8..d3b389293 100644 --- a/src/core/postgres-engine.ts +++ b/src/core/postgres-engine.ts @@ -6153,6 +6153,13 @@ export class PostgresEngine implements BrainEngine { const prefixCondition = slugPrefix ? sql`AND p.slug LIKE ${slugPrefix.replace(/[\\%_]/g, (c) => '\\' + c) + '%'} ESCAPE '\\'` : sql``; + // TIM-37: exclude briefing pages from their own Brain Pulse. The cron + // briefing writes to 90_Briefings/, gets re-ingested, and would otherwise + // top tomorrow's salience as pure self-reference. Suppress unless the + // caller explicitly asked for the briefings/ prefix. + const excludeBriefings = !(slugPrefix && slugPrefix.startsWith('briefings')) + ? sql`AND p.slug NOT LIKE 'briefings/%'` + : sql``; // v0.29.1: third score term via buildRecencyComponentSql. Default // 'flat' = v0.29.0 behavior (1 / (1 + days_old)). 'on' opts into the // per-prefix decay map (concepts/ evergreen, daily/ aggressive, etc.). @@ -6186,6 +6193,7 @@ export class PostgresEngine implements BrainEngine { LEFT JOIN takes t ON t.page_id = p.id AND t.active = TRUE WHERE GREATEST(p.updated_at, COALESCE(p.salience_touched_at, p.updated_at)) >= ${boundaryIso}::timestamptz ${prefixCondition} + ${excludeBriefings} GROUP BY p.id ORDER BY score DESC LIMIT ${limit} diff --git a/test/e2e/salience-pglite.test.ts b/test/e2e/salience-pglite.test.ts index aa959031e..98d46604a 100644 --- a/test/e2e/salience-pglite.test.ts +++ b/test/e2e/salience-pglite.test.ts @@ -112,4 +112,27 @@ describe('v0.29 E2E — getRecentSalience (Garry test)', () => { const rows = await engine.getRecentSalience({ days: 7, slugPrefix: 'nope/does-not-exist/' }); expect(rows).toEqual([]); }); + + // TIM-37: the daily briefing writes to the vault and re-ingests as + // `briefings/`. Without this filter the briefing itself would top + // every subsequent Brain Pulse — self-reference with no signal. + describe('TIM-37 — briefings excluded from their own Brain Pulse', () => { + test('default query hides briefings/* slugs', async () => { + await engine.putPage('briefings/2026-05-19', { + type: 'note', + title: 'Daily Briefing — 2026-05-19', + compiled_truth: 'Auto-generated cron briefing.', + }); + const rows = await engine.getRecentSalience({ days: 7, limit: 50 }); + expect(rows.some(r => r.slug.startsWith('briefings/'))).toBe(false); + }); + + test('explicit slugPrefix=briefings/ still returns them', async () => { + const rows = await engine.getRecentSalience({ days: 7, slugPrefix: 'briefings/' }); + expect(rows.length).toBeGreaterThan(0); + for (const r of rows) { + expect(r.slug.startsWith('briefings/')).toBe(true); + } + }); + }); }); From 292b8b1637f09df66c365d94e884ea25d35ee6ea Mon Sep 17 00:00:00 2001 From: mmekkaoui Date: Thu, 23 Jul 2026 00:22:09 +0200 Subject: [PATCH 05/25] fix(ai): cap llama-server embedding batches at its 32-input request limit (#1281) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit llama.cpp's llama-server rejects /v1/embeddings requests with more inputs than its launch --batch-size (default 32): "batch size 100 > maximum allowed batch size 32". gbrain sends batches of 100, so any page with >32 chunks fails to embed, and embed --stale then trips the Postgres statement_timeout retrying the doomed batches. The existing token-based protection (max_batch_tokens) can't bound item count — N tiny chunks fit under any token budget. Add an optional max_batch_items count cap to EmbeddingTouchpoint, enforced as a hard re-split after the token split in embed(), and set it to 32 on the llama-server recipe (replacing no_batch_cap: true, which wrongly assumed llama.cpp has no per-request item cap). A declared item cap also suppresses the missing-max_batch_tokens startup warning. Co-authored-by: Time Attakc <89218912+time-attack@users.noreply.github.com> --- src/core/ai/gateway.ts | 28 ++++++++++++++- src/core/ai/recipes/llama-server.ts | 9 +++-- src/core/ai/types.ts | 10 ++++++ test/ai/adaptive-embed-batch.test.ts | 36 +++++++++++++++++++ .../no-batch-cap-suppression.serial.test.ts | 14 ++++++-- 5 files changed, 91 insertions(+), 6 deletions(-) diff --git a/src/core/ai/gateway.ts b/src/core/ai/gateway.ts index 8a2674484..970d6b606 100644 --- a/src/core/ai/gateway.ts +++ b/src/core/ai/gateway.ts @@ -599,6 +599,8 @@ function warnRecipesMissingBatchTokens(): void { // LiteLLM proxy, llama-server) — they ship without a static cap because // the cap depends on a user-launched server. Warning is noise for them. if (embedding.no_batch_cap === true) continue; + // A declared item-count cap is a real batch cap — no warning needed. + if (embedding.max_batch_items !== undefined) continue; if (_warnedRecipes.has(recipe.id)) continue; _warnedRecipes.add(recipe.id); // eslint-disable-next-line no-console @@ -1517,10 +1519,17 @@ export async function embed(texts: string[], opts?: EmbedOpts): Promise capBatchItems(b, maxBatchItems)) + : tokenBatches; + const allEmbeddings: Float32Array[] = []; let _embedThrew = false; try { @@ -1596,6 +1605,23 @@ export function splitByTokenBudget( return batches; } +/** + * Split a batch into sub-batches of at most `maxItems` inputs. Enforces a + * hard COUNT cap that the token-budget split can't (many tiny inputs fit + * under any token budget). Used for endpoints like llama.cpp's llama-server + * that reject requests exceeding their launch batch size. + * + * @internal exported for tests; not part of the public gateway API. + */ +export function capBatchItems(texts: string[], maxItems: number): string[][] { + if (maxItems <= 0 || texts.length <= maxItems) return [texts]; + const batches: string[][] = []; + for (let i = 0; i < texts.length; i += maxItems) { + batches.push(texts.slice(i, i + maxItems)); + } + return batches; +} + /** * Returns true if the error looks like a provider batch-token-limit error. * diff --git a/src/core/ai/recipes/llama-server.ts b/src/core/ai/recipes/llama-server.ts index 212bd5478..a51650b27 100644 --- a/src/core/ai/recipes/llama-server.ts +++ b/src/core/ai/recipes/llama-server.ts @@ -35,9 +35,12 @@ export const llamaServer: Recipe = { trust_custom_dims: true, // #2271: user knows the launched model's native dim cost_per_1m_tokens_usd: 0, price_last_verified: '2026-05-10', - // llama-server's batch capacity is set by `--ctx-size` at launch - // time; no static cap to declare. v0.32 (#779). - no_batch_cap: true, + // llama-server enforces a hard request-COUNT cap equal to its launch + // batch size (`--batch-size`, default 32): it rejects requests with + // more inputs with `batch size N > maximum allowed batch size 32`. + // The token-budget split can't bound item count, so cap it here. A + // server launched with a larger `-b` can raise this. v0.32 (#779). + max_batch_items: 32, }, }, /** diff --git a/src/core/ai/types.ts b/src/core/ai/types.ts index 798551eec..8fc785e3d 100644 --- a/src/core/ai/types.ts +++ b/src/core/ai/types.ts @@ -54,6 +54,16 @@ export interface EmbeddingTouchpoint { * `max_batch_tokens` is also set. */ safety_factor?: number; + /** + * Maximum number of inputs per embedding request. Some endpoints enforce a + * hard COUNT cap independent of token budget — notably llama.cpp's + * `llama-server`, which rejects requests with more inputs than its launch + * batch size (e.g. `batch size 100 > maximum allowed batch size 32`). The + * token-budget pre-split cannot bound item count (many tiny chunks fit under + * any token budget), so this is enforced as a separate hard re-split after + * the token split. When unset, no count cap is applied. + */ + max_batch_items?: number; /** * v0.27.1: when true, at least one model in this recipe accepts image * inputs via a multimodal embedding endpoint (e.g. Voyage's diff --git a/test/ai/adaptive-embed-batch.test.ts b/test/ai/adaptive-embed-batch.test.ts index 505c7c82f..144668eca 100644 --- a/test/ai/adaptive-embed-batch.test.ts +++ b/test/ai/adaptive-embed-batch.test.ts @@ -34,6 +34,7 @@ import { resetGateway, embed, splitByTokenBudget, + capBatchItems, isTokenLimitError, __setEmbedTransportForTests, __getShrinkStateForTests, @@ -151,6 +152,41 @@ describe('splitByTokenBudget (pure helper)', () => { }); }); +describe('capBatchItems (hard COUNT cap helper)', () => { + test('batch at or under the cap is returned as a single batch (no copy of contents)', () => { + const texts = ['a', 'b', 'c']; + expect(capBatchItems(texts, 3)).toEqual([texts]); + expect(capBatchItems(texts, 10)).toEqual([texts]); + }); + + test('oversized batch splits into chunks of at most maxItems', () => { + const texts = Array.from({ length: 100 }, (_, i) => `t${i}`); + const result = capBatchItems(texts, 32); + expect(result.map(b => b.length)).toEqual([32, 32, 32, 4]); + expect(result.every(b => b.length <= 32)).toBe(true); + }); + + test('exact multiple splits evenly with no trailing empty batch', () => { + const texts = Array.from({ length: 64 }, (_, i) => `t${i}`); + expect(capBatchItems(texts, 32).map(b => b.length)).toEqual([32, 32]); + }); + + test('order is preserved across the split (concatenation round-trips)', () => { + const texts = Array.from({ length: 70 }, (_, i) => `t${i}`); + expect(capBatchItems(texts, 32).flat()).toEqual(texts); + }); + + test('maxItems <= 0 is a no-op (single batch) — never produces empty/infinite batches', () => { + const texts = ['a', 'b', 'c']; + expect(capBatchItems(texts, 0)).toEqual([texts]); + expect(capBatchItems(texts, -5)).toEqual([texts]); + }); + + test('empty input returns a single empty batch', () => { + expect(capBatchItems([], 32)).toEqual([[]]); + }); +}); + describe('isTokenLimitError (pure helper)', () => { test('matches Voyage error format', () => { expect(isTokenLimitError(VOYAGE_TOKEN_LIMIT_ERROR)).toBe(true); diff --git a/test/ai/no-batch-cap-suppression.serial.test.ts b/test/ai/no-batch-cap-suppression.serial.test.ts index 1241b3b5a..5a5d00f60 100644 --- a/test/ai/no-batch-cap-suppression.serial.test.ts +++ b/test/ai/no-batch-cap-suppression.serial.test.ts @@ -28,8 +28,8 @@ describe('v0.32 #779: no_batch_cap suppresses the missing-max_batch_tokens warni resetGateway(); }); - test('Ollama, LiteLLM, llama-server all declare no_batch_cap: true', () => { - for (const id of ['ollama', 'litellm', 'llama-server']) { + test('Ollama, LiteLLM declare no_batch_cap: true', () => { + for (const id of ['ollama', 'litellm']) { const r = getRecipe(id); expect(r, `${id} not registered`).toBeDefined(); expect( @@ -39,6 +39,16 @@ describe('v0.32 #779: no_batch_cap suppresses the missing-max_batch_tokens warni } }); + test('llama-server declares a hard item-count cap (max_batch_items: 32)', () => { + // llama.cpp enforces a request-COUNT cap equal to its launch --batch-size + // (default 32); declaring max_batch_items both bounds batches AND suppresses + // the missing-max_batch_tokens warning. Replaces the prior no_batch_cap flag. + const r = getRecipe('llama-server'); + expect(r, 'llama-server not registered').toBeDefined(); + expect(r!.touchpoints.embedding?.max_batch_items).toBe(32); + expect(r!.touchpoints.embedding?.no_batch_cap).toBeUndefined(); + }); + test('configureGateway does NOT warn for ollama/litellm/llama-server', () => { warnSpy.mockClear(); resetGateway(); From d43fb631bc2a8d53b8128887306867539d081a73 Mon Sep 17 00:00:00 2001 From: Ryan Ayers Date: Wed, 22 Jul 2026 17:52:59 -0500 Subject: [PATCH 06/25] fix(serve-http): add resource_metadata to WWW-Authenticate per MCP spec + RFC 9728 (#1410) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The HTTP MCP server's 401 responses missed the `resource_metadata` parameter in the WWW-Authenticate header. MCP authorization spec (2025-06-18 draft §5.1) and RFC 9728 require: WWW-Authenticate: Bearer resource_metadata="" MCP-aware OAuth clients (claude.ai, Cursor, etc.) use that URL to find the authorization-server discovery doc without the user manually configuring the issuer. Pre-fix the header shipped only `Bearer error="invalid_token", error_description="..."` and MCP clients silently failed to begin the OAuth flow — symptom on claude.ai's UI was "Couldn't reach the MCP server" even when discovery + /token + /register all responded 200 individually. The `requireBearerAuth` middleware in @modelcontextprotocol/sdk's BearerAuthMiddlewareOptions already supports a `resourceMetadataUrl` parameter. Two call sites (`/mcp` and `/ingest`) now pass it. Verified against a real claude.ai connector attempt: pre-fix the connector showed "Couldn't reach the MCP server" with no OAuth redirect. Post-fix the connector successfully begins the authorization flow. Co-authored-by: Claude Opus 4.7 (1M context) Co-authored-by: Time Attakc <89218912+time-attack@users.noreply.github.com> --- src/commands/serve-http.ts | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/commands/serve-http.ts b/src/commands/serve-http.ts index 2cbae64bc..9b01a183f 100644 --- a/src/commands/serve-http.ts +++ b/src/commands/serve-http.ts @@ -843,6 +843,21 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption // reverse proxies / tunnels; default to localhost for dev. const issuerUrl = new URL(publicUrl || `http://localhost:${port}`); + // MCP authorization spec (2025-06-18 draft §5.1) and RFC 9728 require the + // protected resource server to return its discovery metadata URL in the + // WWW-Authenticate header on 401 responses: + // + // WWW-Authenticate: Bearer resource_metadata="" + // + // Clients (claude.ai, Cursor, every other MCP-aware OAuth client) use that + // URL to find the authorization-server discovery doc + token endpoint + // without the user having to paste those URLs manually. Pre-fix the header + // shipped `Bearer error="invalid_token", ...` with no resource_metadata + // parameter, so MCP clients couldn't begin the OAuth flow from a fresh + // 401 — they would silently fail to connect with a generic "couldn't + // reach the MCP server" error. + const resourceMetadataUrl = `${issuerUrl.toString().replace(/\/$/, '')}/.well-known/oauth-protected-resource`; + // F9: cookie `secure` flag honors both the request's TLS state (req.secure // is set when express trust-proxy lands an X-Forwarded-Proto: https) AND // the operator's declared issuer protocol (so a Cloudflare-tunnel deploy @@ -1601,7 +1616,7 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption res.status(405).json({ jsonrpc: '2.0', error: { code: -32000, message: 'Method not allowed' }, id: null }); }); - app.post('/mcp', requireBearerAuth({ verifier: oauthProvider }), async (req: Request, res: Response) => { + app.post('/mcp', requireBearerAuth({ verifier: oauthProvider, resourceMetadataUrl }), async (req: Request, res: Response) => { const startTime = Date.now(); const authInfo = (req as any).auth as AuthInfo; @@ -1944,7 +1959,7 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption app.post( '/ingest', ingestRateLimiter, - requireBearerAuth({ verifier: oauthProvider, requiredScopes: ['write'] }), + requireBearerAuth({ verifier: oauthProvider, requiredScopes: ['write'], resourceMetadataUrl }), express.raw({ type: '*/*', limit: ingestMaxBytes }), async (req: Request, res: Response) => { const startTime = Date.now(); From 2b020ba2bdf5eb3d4d1ae05c6612c7f5c4812825 Mon Sep 17 00:00:00 2001 From: "Benjamin D. Smith" Date: Thu, 23 Jul 2026 09:17:06 +1000 Subject: [PATCH 07/25] fix(models): dispatch subcommand reads args[0] not args[1] (#1428) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(models): dispatch subcommand reads args[0] not args[1] `gbrain models doctor` silently fell through to the read view instead of running the reachability probe. `runModels` checks `args[1] === 'doctor'`, but the caller — `handleCliOnly(command, subArgs)` in `src/cli.ts:113` — passes `subArgs` (the leading command token already stripped). So inside `runModels`, args[0] is the subcommand. args[1] is undefined. The doctor probe path has been unreachable from the CLI since the handleCliOnly refactor. `gbrain models help` happened to work by falling through to the `--help` flag detection. Two-char fix: `args[1]` → `args[0]` on both branches of the ternary. Verified by manual probe — `gbrain models doctor` now prints "Model reachability probe:" with per-model results (real production brain, 4 touchpoints probed): ``` Model reachability probe: embedding_config ollama:bge-m3 ok (0ms) reranker_config (none) ok (0ms) chat lmstudio:mistralai/magistral-small-2509 unknown (5012ms) [chat(lmstudio:mistralai/magistral-small-2509)] probe timed out after 5s expansion lmstudio:google/gemma-4-e2b ok (535ms) Summary: 3/4 reachable. ``` RECOVERY REBUILD 2026-05-26 of original 20ed0eee. * fix: honor --help before doctor dispatch to avoid running probes on `models doctor --help` Codex review of #1428 flagged that the args[1]→args[0] rewrite regressed `gbrain models doctor --help` into running network probes instead of printing usage. The original args[1]-shaped ternary happened to dodge this by always falling through to the args.includes('--help') branch when args[1] === 'doctor' was false; the new args[0] code checks doctor first, so --help no longer wins. Reorder ternary: `hasHelp` is computed FIRST from (--help / -h / args[0] === 'help'), then the sub is hasHelp ? 'help' : args[0] === 'doctor' ? 'doctor' : 'read'. Addresses codex review P2 on PR #1428. --------- Co-authored-by: Time Attakc <89218912+time-attack@users.noreply.github.com> --- src/commands/models.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/commands/models.ts b/src/commands/models.ts index 8b444370b..b44e8f506 100644 --- a/src/commands/models.ts +++ b/src/commands/models.ts @@ -536,7 +536,20 @@ function shouldSkipProvider(modelStr: string, skip: string[]): boolean { export async function runModels(engine: BrainEngine, args: string[]): Promise { const json = args.includes('--json'); - const sub = args[1] === 'doctor' ? 'doctor' : args[1] === 'help' || args.includes('--help') || args.includes('-h') ? 'help' : 'read'; + // args is `subArgs` from cli.ts `handleCliOnly` — the leading 'models' + // token has already been stripped. The subcommand is at args[0], NOT + // args[1]. Pre-fix this check was `args[1]`, so `gbrain models doctor` + // silently fell through to the read view. The doctor probe path was + // unreachable from the CLI. + // + // --help honored FIRST so `gbrain models doctor --help` shows usage + // instead of running network probes (which would spend tokens or + // exit nonzero when the user only asked for help). Pre-fix the + // args[1] ternary happened to dodge this by always falling through + // to the args.includes('--help') branch; the args[0] rewrite needs + // explicit ordering to preserve that behavior. + const hasHelp = args.includes('--help') || args.includes('-h') || args[0] === 'help'; + const sub = hasHelp ? 'help' : args[0] === 'doctor' ? 'doctor' : 'read'; if (sub === 'help') { process.stdout.write( From 0a757bf7809f67fbd551a6977cb53e821dcd7ec1 Mon Sep 17 00:00:00 2001 From: 0xTim Date: Wed, 22 Jul 2026 16:50:52 -0700 Subject: [PATCH 08/25] fix(entities): thread sourceId through findByTitleFuzzy + skip soft-deleted (#1508) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `findByTitleFuzzy` on both `postgres-engine.ts` and `pglite-engine.ts` has no `source_id` filter and no `deleted_at IS NULL` filter. `tryFuzzyMatch` in `src/core/entities/resolve.ts` got both of those filters via #1436 (v0.41.13.0) for exactly the reasons that apply to its sibling here: fuzzy resolution can suggest cross-source slug candidates that the caller then silently drops at the FK filter (or worse, picks a soft-deleted page). This is the missing twin of #1436. In multi-source brains, the live-mode auto-link resolver invoked from `put_page` (`operations.ts:937`) calls `engine.findByTitleFuzzy` with no scope. When two sources contain pages with similar titles (`people/alice-example` on `source-a`, `people/alice-other` on `source-b`), the fuzzy lookup can return the wrong-source slug, which then fails the downstream `allSlugs` / `addLink` FK filter — the link silently doesn't get created, and from the caller's view the resolver "failed" even though the page existed under the right source. Reproducible with a 2-source PGLite setup + identical-title pages on both sides; the fuzzy call returns a slug whose `source_id` doesn't match the put_page caller's source. - 2-source brains: auto-links between same-title-different-source pages now resolve under the caller's source instead of the wrong neighbor. - Soft-deleted pages can no longer be returned as fuzzy candidates (mirroring the resolve.ts fix from #1436). - 1-source brains: no behavior change. `sourceId` is optional; when omitted the SQL takes the pre-existing unscoped path. - `engine.ts`: add optional 4th `sourceId` param to the `findByTitleFuzzy` interface + JSDoc explaining the scope semantics. - `postgres-engine.ts` / `pglite-engine.ts`: implement the param via a conditional SQL branch that adds `AND source_id = $N AND deleted_at IS NULL` when `sourceId` is set; existing query path unchanged when omitted. - `link-extraction.ts`: add optional `sourceId` to `makeResolver` opts, forward to `findByTitleFuzzy` in step 3 of the resolve chain. - `operations.ts`: pass `opts?.sourceId` to `makeResolver` from the live-mode put_page resolver (the place that already knows the caller's source). - New unit tests in `test/link-extraction.test.ts` (2 cases): - `opts.sourceId` is forwarded to `findByTitleFuzzy` when set. - `opts.sourceId` omitted → `findByTitleFuzzy` receives `undefined` (back-compat). - `bun run typecheck` clean. - `bun test test/link-extraction.test.ts test/entity-resolve.test.ts test/operations.test.ts test/extract.test.ts` — 145/145 pass. Co-authored-by: Time Attakc <89218912+time-attack@users.noreply.github.com> --- src/core/engine.ts | 10 ++++++++++ src/core/link-extraction.ts | 8 ++++++-- src/core/operations.ts | 6 +++++- src/core/pglite-engine.ts | 37 +++++++++++++++++++++++++++--------- src/core/postgres-engine.ts | 35 ++++++++++++++++++++++++++-------- test/link-extraction.test.ts | 37 ++++++++++++++++++++++++++++++++++++ 6 files changed, 113 insertions(+), 20 deletions(-) diff --git a/src/core/engine.ts b/src/core/engine.ts index a389d3091..ef04165b4 100644 --- a/src/core/engine.ts +++ b/src/core/engine.ts @@ -1218,11 +1218,21 @@ export interface BrainEngine { * * Uses the `%` trigram operator (GIN-indexed) + the standard `similarity()` * function. Both engines support pg_trgm (PGLite 0.3+, Postgres always). + * + * `sourceId` constrains the search to a single source and filters out + * soft-deleted pages. Mirrors the same filters `tryFuzzyMatch` in + * `src/core/entities/resolve.ts` got via #1436 (v0.41.13.0). Omit for the + * historical unscoped behavior — live-mode callers that already know + * the source should pass it to avoid cross-source slug suggestions that + * get silently dropped at the FK filter downstream. Batch-mode callers + * (e.g. `gbrain extract`) intentionally omit it to build a cross-source + * resolution map. */ findByTitleFuzzy( name: string, dirPrefix?: string, minSimilarity?: number, + sourceId?: string, ): Promise<{ slug: string; similarity: number } | null>; /** * v0.34.1 (#861 — P0 leak seal): `opts.sourceId` / `opts.sourceIds` diff --git a/src/core/link-extraction.ts b/src/core/link-extraction.ts index 8f27903e6..6ff2f6822 100644 --- a/src/core/link-extraction.ts +++ b/src/core/link-extraction.ts @@ -980,10 +980,14 @@ export function makeResolver( // Step 3: pg_trgm fuzzy title match — both modes. Tries each hint in // order; first hint with a ≥0.55 similarity match wins. If no hints, - // try the whole pages table. + // try the whole pages table. When opts.sourceId is set, the fuzzy + // search is constrained to that source (and skips soft-deleted pages) + // so cross-source slug suggestions don't get silently dropped at the + // FK filter downstream. Mirrors the same scope fix `tryFuzzyMatch` got + // via #1436. const searchHints = hints.length > 0 ? hints : [undefined]; for (const hint of searchHints) { - const match = await engine.findByTitleFuzzy(trimmed, hint, 0.55); + const match = await engine.findByTitleFuzzy(trimmed, hint, 0.55, opts.sourceId); if (match) { cache.set(cacheKey, match.slug); return match.slug; diff --git a/src/core/operations.ts b/src/core/operations.ts index ab2d4cbf1..68f5a56e1 100644 --- a/src/core/operations.ts +++ b/src/core/operations.ts @@ -1153,7 +1153,11 @@ async function runAutoLink( // Live-mode resolver: per-put throwaway cache, pg_trgm + optional search. // Issue #972 (codex [P1]): pass sourceId so basename resolution stays - // within this page's source — no cross-source basename edges. + // within this page's source — no cross-source basename edges. Also scopes + // the fuzzy fallback (findByTitleFuzzy) to the same source the put_page is + // targeting — without it, cross-source slug suggestions get silently dropped + // at the FK filter and the link looks like it failed to resolve. Twin of + // #1436's `tryFuzzyMatch` fix. const resolver = makeResolver(engine, { mode: 'live', sourceId: opts?.sourceId }); // Issue #972: opt-in bare-wikilink basename resolution. Off by default. const globalBasename = await isGlobalBasenameEnabled(engine); diff --git a/src/core/pglite-engine.ts b/src/core/pglite-engine.ts index e11eeb3fb..229322764 100644 --- a/src/core/pglite-engine.ts +++ b/src/core/pglite-engine.ts @@ -2912,22 +2912,41 @@ export class PGLiteEngine implements BrainEngine { name: string, dirPrefix?: string, minSimilarity: number = 0.55, + sourceId?: string, ): Promise<{ slug: string; similarity: number } | null> { // Inline threshold comparison instead of `SET LOCAL pg_trgm.similarity_threshold`. // The GUC only scopes to the current transaction and pglite auto-commits each // .query() call, so the SET LOCAL would be a no-op. Using similarity() >= $N // directly gives predictable behavior. Tie-breaker: sort by slug so re-runs // pick the same winner. + // + // `sourceId` + `deleted_at IS NULL` mirror the filters `tryFuzzyMatch` in + // `src/core/entities/resolve.ts` got via #1436 (v0.41.13.0). Without them, + // fuzzy resolution could suggest cross-source slugs that the caller then + // silently drops at the FK filter — making it look like the match failed + // when in fact it picked the wrong page. const prefixPattern = dirPrefix ? `${dirPrefix}/%` : '%'; - const { rows } = await this.db.query( - `SELECT slug, similarity(title, $1) AS sim - FROM pages - WHERE similarity(title, $1) >= $3 - AND slug LIKE $2 - ORDER BY sim DESC, slug ASC - LIMIT 1`, - [name, prefixPattern, minSimilarity] - ); + const { rows } = sourceId + ? await this.db.query( + `SELECT slug, similarity(title, $1) AS sim + FROM pages + WHERE similarity(title, $1) >= $3 + AND slug LIKE $2 + AND source_id = $4 + AND deleted_at IS NULL + ORDER BY sim DESC, slug ASC + LIMIT 1`, + [name, prefixPattern, minSimilarity, sourceId] + ) + : await this.db.query( + `SELECT slug, similarity(title, $1) AS sim + FROM pages + WHERE similarity(title, $1) >= $3 + AND slug LIKE $2 + ORDER BY sim DESC, slug ASC + LIMIT 1`, + [name, prefixPattern, minSimilarity] + ); if (rows.length === 0) return null; const row = rows[0] as { slug: string; sim: number }; return { slug: row.slug, similarity: row.sim }; diff --git a/src/core/postgres-engine.ts b/src/core/postgres-engine.ts index d3b389293..6deaad784 100644 --- a/src/core/postgres-engine.ts +++ b/src/core/postgres-engine.ts @@ -3082,6 +3082,7 @@ export class PostgresEngine implements BrainEngine { name: string, dirPrefix?: string, minSimilarity: number = 0.55, + sourceId?: string, ): Promise<{ slug: string; similarity: number } | null> { const sql = this.sql; // Use the `similarity()` function directly with an explicit threshold @@ -3094,15 +3095,33 @@ export class PostgresEngine implements BrainEngine { // Tie-breaker: sort by slug after similarity so re-runs return the // same winner when multiple pages score equally (prevents churn // in put_page auto-link reconciliation). + // + // `sourceId` + `deleted_at IS NULL` mirror the filters `tryFuzzyMatch` + // in `src/core/entities/resolve.ts` got via #1436 (v0.41.13.0). Without + // them, fuzzy resolution could suggest cross-source slugs that the + // caller then silently drops at the FK filter in + // `operations.ts:reconcileLinks` (the `allSlugs` filter) — making it + // look like the match failed when in fact it picked the wrong page. const prefixPattern = dirPrefix ? `${dirPrefix}/%` : '%'; - const rows = await sql` - SELECT slug, similarity(title, ${name}) AS sim - FROM pages - WHERE similarity(title, ${name}) >= ${minSimilarity} - AND slug LIKE ${prefixPattern} - ORDER BY sim DESC, slug ASC - LIMIT 1 - `; + const rows = sourceId + ? await sql` + SELECT slug, similarity(title, ${name}) AS sim + FROM pages + WHERE similarity(title, ${name}) >= ${minSimilarity} + AND slug LIKE ${prefixPattern} + AND source_id = ${sourceId} + AND deleted_at IS NULL + ORDER BY sim DESC, slug ASC + LIMIT 1 + ` + : await sql` + SELECT slug, similarity(title, ${name}) AS sim + FROM pages + WHERE similarity(title, ${name}) >= ${minSimilarity} + AND slug LIKE ${prefixPattern} + ORDER BY sim DESC, slug ASC + LIMIT 1 + `; if (rows.length === 0) return null; const row = rows[0] as { slug: string; sim: number }; return { slug: row.slug, similarity: row.sim }; diff --git a/test/link-extraction.test.ts b/test/link-extraction.test.ts index f7c003ab5..9a2bc4f7d 100644 --- a/test/link-extraction.test.ts +++ b/test/link-extraction.test.ts @@ -1236,6 +1236,43 @@ describe('makeResolver — fallback chain', () => { const out = await r.resolveBasenameMatches!('struktura'); expect(out.sort()).toEqual(['notes/struktura', 'struktura']); }); + + test('opts.sourceId is forwarded to findByTitleFuzzy (twin of #1436 fix)', async () => { + // Captures every (name, dirPrefix, minSimilarity, sourceId) call so we + // can assert the resolver threads sourceId through. Without the wire-up, + // findByTitleFuzzy would be called with sourceId=undefined and the SQL + // could return cross-source slug suggestions that the FK filter + // downstream silently drops. + const calls: Array<{ name: string; dirPrefix?: string; minSimilarity?: number; sourceId?: string }> = []; + const engine = { + async getPage() { return null; }, + async findByTitleFuzzy(name: string, dirPrefix?: string, minSimilarity?: number, sourceId?: string) { + calls.push({ name, dirPrefix, minSimilarity, sourceId }); + return null; + }, + async searchKeyword() { return []; }, + } as unknown as BrainEngine; + const r = makeResolver(engine, { mode: 'batch', sourceId: 'src-a' }); + await r.resolve('Alice Example', 'people'); + expect(calls.length).toBeGreaterThan(0); + expect(calls.every(c => c.sourceId === 'src-a')).toBe(true); + }); + + test('opts.sourceId omitted → findByTitleFuzzy receives undefined (back-compat)', async () => { + const calls: Array<{ sourceId?: string }> = []; + const engine = { + async getPage() { return null; }, + async findByTitleFuzzy(_name: string, _dirPrefix?: string, _min?: number, sourceId?: string) { + calls.push({ sourceId }); + return null; + }, + async searchKeyword() { return []; }, + } as unknown as BrainEngine; + const r = makeResolver(engine, { mode: 'batch' }); + await r.resolve('Alice Example', 'people'); + expect(calls.length).toBeGreaterThan(0); + expect(calls.every(c => c.sourceId === undefined)).toBe(true); + }); }); describe('FRONTMATTER_LINK_MAP integrity', () => { From e7ffbc057c7992f901cbeab47aeac0c73ab425e7 Mon Sep 17 00:00:00 2001 From: Thomas Chung Date: Wed, 22 Jul 2026 17:19:29 -0700 Subject: [PATCH 09/25] fix(lint): code-fence-wrap detector and fixer regex now agree (#1597) The code-fence-wrap detector in lintContent used the /m multiline flag, so ^/$ matched start/end of any line. The rule fired on any page that simply contained a ```markdown code block, not only pages wrapped end-to-end. The matching fixer in fixContent has no /m flag, so it can only strip whole-file wrappers. Result: detected issues were marked fixable: true, yet fixContent could never strip them. `gbrain dream` reported "0 fix(es) applied, N remaining" perpetually for the rule. Drops the /m flag from the detector so detector and fixer stay in sync. Whole-file wrapper detection is preserved; inner code blocks no longer trigger the rule. Real-world impact: a brain with 5 docs pages containing markdown examples (skill READMEs, decision registry, journal templates) reports 5 phantom "fixable: true" issues every dream cycle, never converging. After this fix, the dream-cycle lint phase reports only real-and-unfixable issues (missing frontmatter, missing title/type) which is the intended behavior. Two regression tests added in test/lint.test.ts: - Page contains a single inner ```markdown block - Page contains multiple inner ```markdown blocks Both assert no code-fence-wrap issue is reported. The existing "detects wrapping code fences" test (true-positive case) continues to pass; total tests in the file are 18 -> 20. Co-authored-by: Thomas Chung Co-authored-by: Claude Opus 4.7 (1M context) Co-authored-by: Time Attakc <89218912+time-attack@users.noreply.github.com> --- src/commands/lint.ts | 7 ++++++- test/lint.test.ts | 23 +++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/commands/lint.ts b/src/commands/lint.ts index e14e05456..54b81c290 100644 --- a/src/commands/lint.ts +++ b/src/commands/lint.ts @@ -127,7 +127,12 @@ export function lintContent(content: string, filePath: string, opts: LintContent } // Rule: Wrapping code fences (```markdown ... ```) - if (content.match(/^```(?:markdown|md)\s*\n/m) && content.match(/\n```\s*$/m)) { + // Detector intentionally has NO /m flag so ^/$ match start/end of the whole + // file, not inner lines. Keeps detector in sync with fixContent() below, + // which also has no /m flag. Without this, lint reports "fixable" false + // positives on any page that simply contains a ```markdown code block, but + // fixContent can never strip them (its regex only matches whole-file wrappers). + if (content.match(/^```(?:markdown|md)\s*\n/) && content.match(/\n```\s*$/)) { issues.push({ file: filePath, line: 1, rule: 'code-fence-wrap', message: 'Page wrapped in ```markdown code fences (LLM artifact)', diff --git a/test/lint.test.ts b/test/lint.test.ts index ca38b8d23..f843d5859 100644 --- a/test/lint.test.ts +++ b/test/lint.test.ts @@ -32,6 +32,29 @@ describe('lintContent', () => { expect(issues.some(i => i.rule === 'code-fence-wrap')).toBe(true); }); + test('no false positive: page CONTAINS an inner ```markdown code block', () => { + // Real-world case: a docs/SKILL page that shows a markdown example inline. + // Before this fix, the detector used the /m flag so ^/$ matched start/end + // of any line, which fired on any file that simply contained a ```markdown + // line. But fixContent's regex has no /m flag and can only strip whole-file + // wrappers, so the issue was reported as "fixable: true" yet never fixed. + const content = + '---\ntitle: Skill\n---\n\n# Skill\n\nExample input shape:\n\n' + + '```markdown\n# Inner page\nContent.\n```\n\nThat ends the example.\n'; + const issues = lintContent(content, 'test.md'); + expect(issues.filter(i => i.rule === 'code-fence-wrap')).toHaveLength(0); + }); + + test('no false positive: multiple inner ```markdown blocks', () => { + // Documentation pages frequently include several markdown examples. + const content = + '---\ntitle: Examples\n---\n\n# Examples\n\nFirst:\n\n' + + '```markdown\nfoo\n```\n\nSecond:\n\n' + + '```markdown\nbar\n```\n\nDone.\n'; + const issues = lintContent(content, 'test.md'); + expect(issues.filter(i => i.rule === 'code-fence-wrap')).toHaveLength(0); + }); + test('detects placeholder dates', () => { const content = '---\ntitle: Test\ntype: person\ncreated: YYYY-MM-DD\n---\n\n# Test'; const issues = lintContent(content, 'test.md'); From 56ccc14bcc9088ef6a88403c9ccdfcdbfcdfe94a Mon Sep 17 00:00:00 2001 From: Ryan Ayers Date: Wed, 22 Jul 2026 19:56:33 -0500 Subject: [PATCH 10/25] fix(code-def): surface method/constructor/field/struct definitions (#1628) DEF_TYPES listed only canonical symbol-type names (function, class, interface, ...). But normalizeSymbolType in the code chunker canonicalizes only some tree-sitter node types and lets the rest fall through type.replace(/_/g, ' '). So method_declaration is stored as 'method declaration', struct_specifier as 'struct specifier', protocol_declaration as 'protocol declaration'. None were in DEF_TYPES, so code-def returned 0 hits for every method, constructor, field, C struct, and Swift protocol. The plain 'struct' entry never matched either. Add the fallthrough definition forms. Read-path only; no reindex needed (0 -> N on existing indexes). Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: Time Attakc <89218912+time-attack@users.noreply.github.com> --- src/commands/code-def.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/commands/code-def.ts b/src/commands/code-def.ts index 7cb3690a2..26573cae8 100644 --- a/src/commands/code-def.ts +++ b/src/commands/code-def.ts @@ -37,9 +37,19 @@ export async function findCodeDef( // trigger) are first-class definitions in the SQL sense. The chunker's // normalizeSymbolType maps create_table → 'table' etc, so adding the SQL // kinds here is what makes `gbrain code-def users` work against SQL. + // Method-level + member definitions. normalizeSymbolType only canonicalizes + // some node types; the rest fall through `type.replace(/_/g, ' ')`, so + // tree-sitter's method_declaration → 'method declaration', struct_specifier → + // 'struct specifier', protocol_declaration → 'protocol declaration', etc. + // Without these, code-def is blind to every method, constructor, field, C + // struct, and Swift protocol — which is most of an OO codebase. The plain + // 'struct' entry above never matched for the same reason (C emits the + // 'struct specifier' fallback form). const DEF_TYPES = [ 'function', 'class', 'interface', 'type', 'enum', 'struct', 'trait', 'module', 'contract', 'table', 'view', 'index', 'procedure', 'schema', 'database', 'trigger', + 'method declaration', 'method definition', 'constructor declaration', + 'field declaration', 'field definition', 'struct specifier', 'protocol declaration', ]; const params: unknown[] = [symbol, limit]; let whereLang = ''; From 44cae623240e3e1f44aca1519fabe56c81652181 Mon Sep 17 00:00:00 2001 From: Alex Hawkins Date: Wed, 22 Jul 2026 18:26:48 -0700 Subject: [PATCH 11/25] fix(pglite): guard putPage against zero-row RETURNING (#1649) PGLite can return zero rows from INSERT ... ON CONFLICT DO UPDATE ... RETURNING in no-op/trigger edge cases. The previous code called rowToPage(rows[0]) unconditionally, so rows[0] was undefined and rowToPage threw "undefined is not an object (evaluating 'row.deleted_at')", which aborted the import and silently skipped the file during sync. getPage() already has the empty-rows guard; putPage() was missing the parallel one. The row was in fact written by the upsert, so re-read it via getPage() instead of crashing. On a real monorepo index this recovered ~19% of files (985/5148) that were failing to embed. Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: Time Attakc <89218912+time-attack@users.noreply.github.com> --- src/core/pglite-engine.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/core/pglite-engine.ts b/src/core/pglite-engine.ts index 229322764..f74e4e964 100644 --- a/src/core/pglite-engine.ts +++ b/src/core/pglite-engine.ts @@ -1060,6 +1060,16 @@ export class PGLiteEngine implements BrainEngine { RETURNING id, source_id, slug, type, title, compiled_truth, timeline, frontmatter, content_hash, created_at, updated_at, effective_date, effective_date_source, import_filename, source_kind, source_uri, ingested_via, ingested_at`, [sourceId, slug, page.type, pageKind, page.title, page.compiled_truth, page.timeline || '', JSON.stringify(frontmatter), hash, effectiveDate, effectiveDateSource, importFilename, chunkerVersion, sourcePath, sourceKind, sourceUri, ingestedVia, ingestedAt] ); + // PGLite can return zero rows from INSERT ... ON CONFLICT DO UPDATE ... + // RETURNING in no-op/trigger edge cases, which made rowToPage(undefined) + // throw "undefined is not an object (evaluating 'row.deleted_at')" and + // skip the file during sync. The row WAS written, so re-read instead of + // crashing. + if (rows.length === 0) { + const reread = await this.getPage(slug, { sourceId }); + if (reread) return reread; + throw new Error(`putPage: RETURNING produced no row for ${sourceId}/${slug}`); + } return rowToPage(rows[0] as Record); } From e1526bfebe365db9b00d6a8d9ab61eb3faecb3d9 Mon Sep 17 00:00:00 2001 From: Om Mishra <152969928+howwohmm@users.noreply.github.com> Date: Thu, 23 Jul 2026 07:16:29 +0530 Subject: [PATCH 12/25] fix(think): render the Gaps section once instead of twice (#1662) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gbrain think printed "## Gaps" twice: the synthesis prompt asked the model for a Gaps section inside the answer body AND a separate structured gaps array, then both render paths printed both — the CLI human output (src/commands/think.ts) and the --save page (persistSynthesis in src/core/think/index.ts). Make the structured gaps array the single source. The prompt now routes gaps into the array, not an answer-body section. New exported stripGapsSection(answer) defensively removes any "## Gaps" section a model still emits (any heading level, case-insensitive, bounded by the next same/higher heading); both render sites call it, so the dedup is structural rather than dependent on the model obeying the prompt. Adds test/think-gaps.test.ts (hermetic): strip helper across heading levels / case / no-section / mid-document / false-match, the one-render-only repro, and the prompt contract. Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: Time Attakc <89218912+time-attack@users.noreply.github.com> --- src/commands/think.ts | 4 +- src/core/think/index.ts | 36 +++++++++++++++- src/core/think/prompt.ts | 12 +++--- test/think-gaps.test.ts | 90 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 133 insertions(+), 9 deletions(-) create mode 100644 test/think-gaps.test.ts diff --git a/src/commands/think.ts b/src/commands/think.ts index e3c10882f..9ce9ee099 100644 --- a/src/commands/think.ts +++ b/src/commands/think.ts @@ -6,7 +6,7 @@ * degrades to gather-only output with a warning if missing. */ import type { BrainEngine } from '../core/engine.ts'; -import { runThink, persistSynthesis } from '../core/think/index.ts'; +import { runThink, persistSynthesis, stripGapsSection } from '../core/think/index.ts'; import { loadConfig, isThinClient } from '../core/config.ts'; import { callRemoteTool, unpackToolResult } from '../core/mcp-client.ts'; @@ -157,7 +157,7 @@ prints what would have been the input (exit 0). // Human-readable output console.log(`# ${question}\n`); - console.log(result.answer); + console.log(stripGapsSection(result.answer)); console.log(''); if (result.gaps.length > 0) { console.log('## Gaps'); diff --git a/src/core/think/index.ts b/src/core/think/index.ts index e11d7475f..f06ab59dc 100644 --- a/src/core/think/index.ts +++ b/src/core/think/index.ts @@ -553,6 +553,40 @@ export async function runThink( }; } +/** + * Strip a "## Gaps" section from an answer body. + * + * `think` returns gaps in the structured `gaps` array, which the CLI and the + * persisted synthesis page render exactly once. The system prompt also used to + * ask for a "Gaps" section inside the answer prose, so a model that still emits + * one would make the output show "## Gaps" twice — once from the prose, once + * from the structured array. This removes the prose section so the structured + * array stays the single source of truth. + * + * Matches a heading line `## Gaps` (level 2-6, case-insensitive) and removes it + * through the next heading of the same-or-higher level, or end of string. + * Returns the input unchanged when there is no such section. + */ +export function stripGapsSection(answer: string): string { + if (!answer) return answer; + const lines = answer.split('\n'); + let start = -1; + let level = 0; + for (let i = 0; i < lines.length; i++) { + const m = /^(#{2,6})\s+gaps\s*$/i.exec(lines[i]); + if (m) { start = i; level = m[1].length; break; } + } + if (start === -1) return answer; + let end = lines.length; + for (let i = start + 1; i < lines.length; i++) { + const h = /^(#{1,6})\s+\S/.exec(lines[i]); + if (h && h[1].length <= level) { end = i; break; } + } + const kept = [...lines.slice(0, start), ...lines.slice(end)].join('\n'); + // Drop trailing blank lines left by removing a trailing section. + return kept.replace(/\s+$/, ''); +} + /** * Persist a synthesis page + its evidence. Returns the saved slug. * Synthesis pages are written under `synthesis/-.md`. @@ -582,7 +616,7 @@ export async function persistSynthesis( const body = [ `# ${result.question}`, '', - result.answer, + stripGapsSection(result.answer), '', result.gaps.length > 0 ? '## Gaps\n\n' + result.gaps.map(g => `- ${g}`).join('\n') : '', ].filter(Boolean).join('\n'); diff --git a/src/core/think/prompt.ts b/src/core/think/prompt.ts index 7107ef729..06a16f84a 100644 --- a/src/core/think/prompt.ts +++ b/src/core/think/prompt.ts @@ -52,19 +52,19 @@ Hard rules: rather than asserting it as established. Confidence is part of the data. - If two takes contradict (different holders, opposite claims), surface BOTH in a "Conflicts" section. Never silently pick one. -- If you cannot answer because the brain doesn't contain the relevant data, say so in the - "Gaps" section. List the specific missing pieces. Do not make up answers. +- If the brain doesn't contain data needed to answer, do NOT make it up. Record each + missing piece in the structured "gaps" array (below), not as a section in the answer prose. - Never instruct the user (no "you should" / "I recommend X"). The brain reports; the user decides. - Output MUST be valid JSON matching the schema below. No prose outside JSON. Output schema: { - "answer": "", + "answer": "", "citations": [ {"page_slug": "people/alice-example", "row_num": 3, "citation_index": 1}, {"page_slug": "companies/acme-example", "row_num": null, "citation_index": 2} ], - "gaps": ["specific missing data point 1", "specific missing data point 2"] + "gaps": ["a specific, self-contained missing-or-stale data point, citing the [slug] where relevant", "another specific gap"] } The "row_num" field is required for take citations and MUST be null for page-only citations.`; @@ -83,7 +83,7 @@ export function buildThinkSystemPrompt(opts: ThinkSystemPromptOpts = {}): string lines.push(`\nThis is a temporal question. Order key claims chronologically when it helps the reader.`); } if (opts.willSave) { - lines.push(`\nThis synthesis will be persisted as a brain page. Aim for completeness — cover Answer, Conflicts, and Gaps thoroughly.`); + lines.push(`\nThis synthesis will be persisted as a brain page. Aim for completeness — cover the Answer and any Conflicts thoroughly, and list every missing piece in the structured "gaps" array.`); } if (opts.withCalibration) { lines.push( @@ -92,7 +92,7 @@ export function buildThinkSystemPrompt(opts: ThinkSystemPromptOpts = {}): string lines.push(`- Name both the user's PRIOR (default reasoning) AND the COUNTER-PRIOR from their hedged-domain self.`); lines.push(`- Reference active bias tags by name when relevant ("this fits the over-confident-geography pattern").`); lines.push(`- Do NOT silently substitute the debiased answer. ALWAYS surface both priors transparently.`); - lines.push(`- Track-record sentences belong in a "Calibration" section in the answer body, between Conflicts and Gaps.`); + lines.push(`- Track-record sentences belong in a "Calibration" section in the answer body, after the Conflicts section (if present).`); } return lines.join('\n'); } diff --git a/test/think-gaps.test.ts b/test/think-gaps.test.ts new file mode 100644 index 000000000..3b1660700 --- /dev/null +++ b/test/think-gaps.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, test } from 'bun:test'; +import { stripGapsSection } from '../src/core/think/index.ts'; +import { buildThinkSystemPrompt } from '../src/core/think/prompt.ts'; + +// `gbrain think` returns gaps in the structured `gaps` array, which both the +// CLI (`src/commands/think.ts`) and the persisted synthesis page +// (`persistSynthesis`) render exactly once. Older prompts also asked for a +// "Gaps" section inside the answer prose, so a model that still emits one made +// the output print "## Gaps" twice. `stripGapsSection` removes the prose +// section so the structured array is the single source of truth. + +describe('stripGapsSection', () => { + test('removes a trailing "## Gaps" section', () => { + const answer = 'The answer with a claim [people/alice].\n\n## Gaps\n- no update since 2026-03-22 [projects/acme]\n- pricing not recorded'; + const out = stripGapsSection(answer); + expect(out).not.toContain('## Gaps'); + expect(out).not.toContain('no update since'); + expect(out).toContain('The answer with a claim [people/alice].'); + }); + + test('removes a level-3 "### Gaps" section', () => { + const out = stripGapsSection('Body text.\n\n### Gaps\n- missing thing'); + expect(out).not.toMatch(/#+\s+Gaps/i); + expect(out).toBe('Body text.'); + }); + + test('is case-insensitive', () => { + expect(stripGapsSection('Body.\n\n## GAPS\n- x')).toBe('Body.'); + expect(stripGapsSection('Body.\n\n## gaps\n- x')).toBe('Body.'); + }); + + test('returns the answer unchanged when there is no Gaps section', () => { + const answer = 'Just an answer.\n\n## Conflicts\n- a vs b'; + expect(stripGapsSection(answer)).toBe(answer); + }); + + test('does not match a heading that merely starts with "Gaps"', () => { + const answer = 'Body.\n\n## Gaps in the coverage\n- this is real content'; + expect(stripGapsSection(answer)).toBe(answer); + }); + + test('stops at the next same-or-higher heading (preserves later content)', () => { + const answer = 'Intro.\n\n## Gaps\n- missing x\n\n## Sources\n- [a]'; + const out = stripGapsSection(answer); + expect(out).not.toContain('missing x'); + expect(out).toContain('## Sources'); + expect(out).toContain('- [a]'); + }); + + test('handles empty / falsy input', () => { + expect(stripGapsSection('')).toBe(''); + }); + + test('the bug repro: strip + structured render yields exactly one "## Gaps"', () => { + // Mirrors the render in src/commands/think.ts: print the (stripped) answer, + // then append one "## Gaps" block from the structured `gaps` array. + const answer = 'Answer prose [people/alice].\n\n## Gaps\n- the prose gap, slightly different wording'; + const gaps = ['the structured gap']; + const rendered = + stripGapsSection(answer) + '\n\n## Gaps\n' + gaps.map((g) => `- ${g}`).join('\n'); + expect((rendered.match(/## Gaps/g) ?? []).length).toBe(1); + expect(rendered).toContain('the structured gap'); + }); +}); + +describe('buildThinkSystemPrompt — gaps go in the structured array, not the answer body', () => { + test('the answer schema no longer lists "Gaps" as a body section', () => { + const out = buildThinkSystemPrompt({}); + expect(out).not.toContain('Sections: Answer, Conflicts (optional), Gaps'); + expect(out).toContain('gaps belong in the gaps array'); + }); + + test('still requires the structured "gaps" array', () => { + const out = buildThinkSystemPrompt({}); + expect(out).toContain('"gaps"'); + }); + + test('preserves the Conflicts section and the Hard rules', () => { + const out = buildThinkSystemPrompt({}); + expect(out).toContain('Conflicts'); + expect(out).toContain('Hard rules:'); + expect(out).toContain('Cite EVERY substantive claim'); + }); + + test('willSave mode routes gaps to the structured array (no body Gaps section)', () => { + const out = buildThinkSystemPrompt({ willSave: true }); + expect(out).not.toContain('cover Answer, Conflicts, and Gaps thoroughly'); + expect(out).toContain('structured "gaps" array'); + }); +}); From 8837bfe5f21bc353f7426b07eae4361946b8f555 Mon Sep 17 00:00:00 2001 From: Lubos Buracinsky <159481718+lubosxyz@users.noreply.github.com> Date: Thu, 23 Jul 2026 03:46:34 +0200 Subject: [PATCH 13/25] fix(chunker): cap oversized code chunks so they stay embeddable (#1675) splitLargeNode can only break up a node that exposes a `body` with >= 2 named children. A node without one -- a giant object/array literal, a single huge assignment, a massive template literal -- is emitted whole. On real source that yields a chunk far larger than the embedder's context window; the embedder then rejects it ("input exceeds context length") and it is never embedded. Example: a 372 KB service file produced 113 chunks, one a single 281 KB (~70k-token) node -> permanently unembedded. Add a final safety-net pass (capOversizedChunks) that recursively re-splits any chunk over a token budget (default 2000, configurable via maxChunkTokens), with a hard character split as a last resort for no-whitespace content (minified one-liners). Normal files are untouched. Verified: that 372 KB file now yields 188 chunks, max ~1.5k tokens, zero oversized; a small file is unchanged. Co-authored-by: Claude Opus 4.8 (1M context) --- src/core/chunkers/code.ts | 81 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 2 deletions(-) diff --git a/src/core/chunkers/code.ts b/src/core/chunkers/code.ts index 1ff1d2efd..5578a290a 100644 --- a/src/core/chunkers/code.ts +++ b/src/core/chunkers/code.ts @@ -168,6 +168,15 @@ export interface CodeChunkOptions { largeChunkThresholdTokens?: number; fallbackChunkSizeWords?: number; fallbackOverlapWords?: number; + /** + * Hard upper bound (estimated tokens) on any single emitted chunk. A node + * the AST splitter can't break up (a giant object/array literal, a single + * huge assignment, a massive template literal) would otherwise be emitted + * whole and rejected by the embedder ("input exceeds context length"). + * Chunks over this budget are recursively re-split. Default 2000 fits the + * smallest common embedder context (e.g. nomic-embed-text, 2048). + */ + maxChunkTokens?: number; } /** @@ -549,6 +558,7 @@ export function parseWithTimeout( } const DEFAULT_CHUNKER_TIMEOUT_MS = 30_000; +const DEFAULT_MAX_CHUNK_TOKENS = 2000; function resolveChunkerTimeoutMs(): number { const raw = process.env.GBRAIN_CHUNKER_TIMEOUT_MS; @@ -706,9 +716,9 @@ export async function chunkCodeTextFull( } if (chunks.length === 0) { - return { chunks: fallbackChunks(source, filePath, language, opts), edges: rawEdges }; + return { chunks: capOversizedChunks(fallbackChunks(source, filePath, language, opts), filePath, language, opts), edges: rawEdges }; } - return { chunks: mergeSmallSiblings(chunks, chunkTarget), edges: rawEdges }; + return { chunks: capOversizedChunks(mergeSmallSiblings(chunks, chunkTarget), filePath, language, opts), edges: rawEdges }; } catch { return { chunks: fallbackChunks(source, filePath, language, opts), edges: [] }; } finally { @@ -814,6 +824,73 @@ function buildMergedChunk(group: CodeChunk[], index: number): CodeChunk { }; } +/** + * Final safety net: guarantee no emitted chunk exceeds the embedder's context + * budget. tree-sitter splitting (splitLargeNode) can only break up a node that + * exposes a `body` with >= 2 named children. A node without one — a giant + * object/array literal, a single huge assignment, a massive template literal — + * is emitted whole, producing a chunk far larger than the embedder accepts. + * The embedder then rejects it ("input exceeds context length") and the chunk + * is never embedded. Recursively re-split any over-budget chunk; fall back to a + * hard character split for pathological no-whitespace content (e.g. a minified + * one-liner) where word/line splitting can't get under budget. + */ +function capOversizedChunks( + chunks: CodeChunk[], + filePath: string, + language: SupportedCodeLanguage, + opts: CodeChunkOptions, +): CodeChunk[] { + const cap = opts.maxChunkTokens ?? DEFAULT_MAX_CHUNK_TOKENS; + if (!chunks.some((c) => estimateTokens(c.text) > cap)) return chunks; + const out: CodeChunk[] = []; + for (const c of chunks) { + if (estimateTokens(c.text) <= cap) { + out.push({ ...c, index: out.length }); + continue; + } + // Strip the structured header ("[Lang] path:N-M symbol\n\n") so the splitter + // works on the raw body; buildChunk re-adds a header to each piece. + const body = c.text.replace(/^\[[^\]]+\] [^\n]+\n\n/, ''); + for (const piece of splitToTokenBudget(body, cap, opts)) { + if (!piece.trim()) continue; + out.push(buildChunk({ + body: piece, + filePath, + language, + symbolName: c.metadata.symbolName, + symbolType: c.metadata.symbolType, + startLine: c.metadata.startLine, + endLine: c.metadata.endLine, + index: out.length, + parentSymbolPath: c.metadata.parentSymbolPath, + })); + } + } + return out; +} + +/** Split `text` into pieces each estimated <= cap tokens. Word/line-aware + * (recursiveChunk) first; a hard character split is the last resort for + * content with no whitespace to break on. */ +function splitToTokenBudget(text: string, cap: number, opts: CodeChunkOptions): string[] { + const out: string[] = []; + const pieces = recursiveChunk(text, { + chunkSize: opts.fallbackChunkSizeWords ?? 300, + chunkOverlap: opts.fallbackOverlapWords ?? 50, + }).map((p) => p.text); + for (const piece of pieces) { + if (estimateTokens(piece) <= cap) { + out.push(piece); + continue; + } + // ~3.5 chars/token is a conservative cl100k estimate for source text. + const charBudget = Math.max(1, Math.floor(cap * 3.5)); + for (let i = 0; i < piece.length; i += charBudget) out.push(piece.slice(i, i + charBudget)); + } + return out; +} + // ---------- Internals ---------- function fallbackChunks( From 8a5296f3cbab997fb9452f986488aefcfbe9b14e Mon Sep 17 00:00:00 2001 From: The Lord Argus Date: Thu, 23 Jul 2026 07:16:39 +0530 Subject: [PATCH 14/25] fix: merge provider base URL config from DB (#1676) Co-authored-by: The Lord Argus <215461619+TheLordArgus@users.noreply.github.com> --- src/core/config.ts | 34 ++++++++++++++++++++- test/cli-multimodal-integration.test.ts | 40 ++++++++++++++++++++++++- test/loadConfig-merge.test.ts | 29 ++++++++++++++++++ 3 files changed, 101 insertions(+), 2 deletions(-) diff --git a/src/core/config.ts b/src/core/config.ts index 4ca00cdc9..e81954886 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -620,7 +620,10 @@ export function loadConfig(): GBrainConfig | null { * size the schema and must be stable across engine connect. */ export async function loadConfigWithEngine( - engine: { getConfig(key: string): Promise }, + engine: { + getConfig(key: string): Promise; + listConfigKeys?(prefix: string): Promise; + }, base?: GBrainConfig | null, ): Promise { // Codex /ship finding #3: when there's no file config AND no env DB URL, @@ -657,11 +660,31 @@ export async function loadConfigWithEngine( return undefined; } } + async function dbPrefixMap(prefix: string): Promise | undefined> { + if (typeof engine.listConfigKeys !== 'function') return undefined; + let keys: string[]; + try { + keys = await engine.listConfigKeys(prefix); + } catch { + return undefined; + } + + const out: Record = {}; + for (const key of keys.sort()) { + if (!key.startsWith(prefix)) continue; + const leaf = key.slice(prefix.length); + if (!leaf) continue; + const value = await dbStr(key); + if (value !== undefined) out[leaf] = value; + } + return Object.keys(out).length > 0 ? out : undefined; + } const dbMultimodal = await dbBool('embedding_multimodal'); const dbMultimodalModel = await dbStr('embedding_multimodal_model'); const dbOcr = await dbBool('embedding_image_ocr'); const dbOcrModel = await dbStr('embedding_image_ocr_model'); + const dbProviderBaseUrls = await dbPrefixMap('provider_base_urls.'); // v0.36 (D7) — embedding-column registry merge. Stored as JSON string in // the config table. Parse + shape-check here; full registry validation // (regex on keys, type/dim/provider field shapes) runs in the resolver at @@ -685,6 +708,15 @@ export async function loadConfigWithEngine( if (merged.embedding_image_ocr_model === undefined && dbOcrModel !== undefined) { merged.embedding_image_ocr_model = dbOcrModel; } + if (dbProviderBaseUrls !== undefined) { + const next = { ...(merged.provider_base_urls ?? {}) }; + for (const [providerId, baseUrl] of Object.entries(dbProviderBaseUrls)) { + if (next[providerId] === undefined) next[providerId] = baseUrl; + } + if (Object.keys(next).length > 0) { + merged.provider_base_urls = next; + } + } if (merged.embedding_columns === undefined && dbEmbeddingColumns !== undefined) { try { const parsed = JSON.parse(dbEmbeddingColumns); diff --git a/test/cli-multimodal-integration.test.ts b/test/cli-multimodal-integration.test.ts index 640ecf31c..b4894faad 100644 --- a/test/cli-multimodal-integration.test.ts +++ b/test/cli-multimodal-integration.test.ts @@ -8,13 +8,15 @@ // // PGLite-only: in-memory engine, no DATABASE_URL needed. -import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test'; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from 'bun:test'; import { PGLiteEngine } from '../src/core/pglite-engine.ts'; import { loadConfigWithEngine, type GBrainConfig } from '../src/core/config.ts'; import { + __setRerankTransportForTests, configureGateway, getEmbeddingModel, getMultimodalModel, + rerank, resetGateway, } from '../src/core/ai/gateway.ts'; import type { AIGatewayConfig } from '../src/core/ai/types.ts'; @@ -52,10 +54,16 @@ afterAll(async () => { beforeEach(async () => { resetGateway(); + __setRerankTransportForTests(null); // Clear any prior config rows so tests are independent. setConfig with // empty string is treated as undefined by loadConfigWithEngine (per // dbStr semantics), so this is safe to call between tests. await engine.setConfig('embedding_multimodal_model', ''); + await engine.setConfig('provider_base_urls.llama-server-reranker', ''); +}); + +afterEach(() => { + __setRerankTransportForTests(null); }); describe('cli connectEngine — embedding_multimodal_model DB→gateway plumbing', () => { @@ -122,4 +130,34 @@ describe('cli connectEngine — embedding_multimodal_model DB→gateway plumbing expect(getEmbeddingModel()).toBe('openai:text-embedding-3-large'); expect(getMultimodalModel()).toBeUndefined(); }); + + test('DB-set provider_base_urls.llama-server-reranker flows to gateway.rerank URL', async () => { + await engine.setConfig('provider_base_urls.llama-server-reranker', 'http://127.0.0.1:8091/v1'); + + const baseConfig: GBrainConfig = { + engine: 'pglite', + embedding_model: 'openai:text-embedding-3-large', + embedding_dimensions: 1536, + }; + + const merged = await loadConfigWithEngine(engine, baseConfig); + configureGateway(buildGatewayConfig(merged!)); + + let capturedUrl = ''; + __setRerankTransportForTests(async (url) => { + capturedUrl = url; + return new Response(JSON.stringify({ results: [{ index: 0, relevance_score: 0.9 }] }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }); + + await rerank({ + query: 'q', + documents: ['d'], + model: 'llama-server-reranker:qwen3-reranker-4b', + }); + + expect(capturedUrl).toBe('http://127.0.0.1:8091/v1/rerank'); + }); }); diff --git a/test/loadConfig-merge.test.ts b/test/loadConfig-merge.test.ts index 7a8ab08af..e5a861960 100644 --- a/test/loadConfig-merge.test.ts +++ b/test/loadConfig-merge.test.ts @@ -9,6 +9,7 @@ import { loadConfigWithEngine, type GBrainConfig } from '../src/core/config.ts'; interface FakeEngine { getConfig(key: string): Promise; + listConfigKeys?(prefix: string): Promise; } function makeEngine(map: Record): FakeEngine { @@ -16,6 +17,9 @@ function makeEngine(map: Record): FakeEngine async getConfig(key: string) { return map[key]; }, + async listConfigKeys(prefix: string) { + return Object.keys(map).filter(key => key.startsWith(prefix)); + }, }; } @@ -92,6 +96,31 @@ describe('loadConfigWithEngine (Phase 4 / F3)', () => { expect(merged?.embedding_image_ocr).toBe(true); }); + test('DB provider_base_urls. fills the gateway base URL map', async () => { + const base: GBrainConfig = { engine: 'pglite' }; + const engine = makeEngine({ + 'provider_base_urls.llama-server-reranker': 'http://127.0.0.1:8091/v1', + }); + const merged = await loadConfigWithEngine(engine, base); + expect(merged?.provider_base_urls?.['llama-server-reranker']).toBe('http://127.0.0.1:8091/v1'); + }); + + test('provider_base_urls merge is per-provider: file value wins and DB fills siblings', async () => { + const base: GBrainConfig = { + engine: 'pglite', + provider_base_urls: { + 'llama-server-reranker': 'http://file.example/v1', + }, + }; + const engine = makeEngine({ + 'provider_base_urls.llama-server-reranker': 'http://db.example/v1', + 'provider_base_urls.openrouter': 'http://openrouter.example/v1', + }); + const merged = await loadConfigWithEngine(engine, base); + expect(merged?.provider_base_urls?.['llama-server-reranker']).toBe('http://file.example/v1'); + expect(merged?.provider_base_urls?.openrouter).toBe('http://openrouter.example/v1'); + }); + test('engine.getConfig throwing is non-fatal — file/env config still returned', async () => { const base: GBrainConfig = { engine: 'pglite', From 2c96787867e72379b6fdb4b0c9281d144d4c9add Mon Sep 17 00:00:00 2001 From: Aurora Capital <201698397+auroracapital@users.noreply.github.com> Date: Thu, 23 Jul 2026 01:46:44 +0000 Subject: [PATCH 15/25] =?UTF-8?q?test(e2e):=20harden=20suite=20=E2=80=94?= =?UTF-8?q?=20kill=20flakes,=20no-op=20assertions,=20cross-test=20coupling?= =?UTF-8?q?=20(#1704)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(e2e): drop flaky wall-clock bounds in minions-resilience The runaway-dead-letter and cascade-kill tests asserted tight real-clock upper bounds (<2000ms, <3000ms) on top of already-complete terminal-state checks. Those bounds carry no correctness signal — the dead/cancelled status and abortedChildren==10 assertions fully prove behavior — and flake on loaded CI runners where the stall/timeout sweep cadence varies. Removed both bounds, kept the diagnostic values, de-promised the test titles. * test(e2e): kill order-dependence + no-op assertions in mechanical - traverse_graph: self-contained (re-adds its own idempotent link) and asserts the linked company is reachable, instead of depending on a prior it() and only checking array shape. - file_list-without-slug: seeds its own >100 rows instead of relying on the previous test's 150 surviving in the DB; asserts the cap is exercised. - precision@5: add a loose floor (every known-item query surfaces >=1 truth doc in top-5) so a 0% retrieval regression no longer passes silently. - get_health: assert value bounds (page_count==16, embed_coverage 0..1) not just typeof; get_chunks: assert non-empty text, numeric non-decreasing chunk_index, and that the page name appears, instead of toBeTruthy on chunks[0]. * test(e2e): strengthen graph/search quality assertions + close coverage gaps graph-quality: truncate+reseed 'config' in truncateAll (kills config leak where a setConfig test throwing before its finally bleeds into later tests); replace toBeGreaterThan(0) link/timeline floors with fixture-derived minimums; assert exact attendee slugs are 'attended' instead of a vacuous .every; pin autoLinks.created to the provable 2 (Alice+Acme); add direction out/both + depth:2 multi-hop and a cycle-safety (A->B->A terminates) test. search-quality: fix the vacuous detail=low vector test; assert pedro returns >=2 chunks; assert detail=high includes the timeline chunk; add empty-query and zero-vector no-throw edge tests. * test(e2e): self-contain multi-source sync test + assert ledger cascade Break the sequential dependency where 'performSync no sourceId' relied on a prior test writing sync.repo_path — it now sets its own config. Add the missing file_migration_ledger COUNT(*)==0 cascade assertion. Tighten the source_id default check from toContain('default') to exact "'default'::text". * test(e2e): make migration-flow HOME/PATH swap throw-safe The suite repoints process.env.HOME/PATH to a temp dir and only restored them in afterAll, so a mid-test throw left HOME dead for the rest of the bun process and silently broke sibling suites. Wrap each test body in try/finally restore + a defensive restore at the top of beforeEach. * test(e2e): loud-skip jsonb-roundtrip + doctor-progress Both skipped silently with no DATABASE_URL, giving zero signal the regression guard never ran. Add the console.log skip line matching the sibling e2e files. * test(e2e): robust check-update contract + find_orphans tool coverage upgrade: the 'no-releases' test hard-asserted update_available===false, which flips to failing the moment the repo has a real release. Assert the JSON contract shape (boolean update_available, current_version===VERSION, typed optional fields) instead. mcp: add find_orphans to the asserted generated tool names. --- test/e2e/doctor-progress.test.ts | 4 + test/e2e/graph-quality.test.ts | 76 +++++++- test/e2e/jsonb-roundtrip.test.ts | 4 + test/e2e/mcp.test.ts | 1 + test/e2e/mechanical.test.ts | 71 +++++++- test/e2e/migration-flow.test.ts | 260 ++++++++++++++++------------ test/e2e/minions-resilience.test.ts | 18 +- test/e2e/multi-source.test.ts | 26 ++- test/e2e/search-quality.test.ts | 17 +- test/e2e/upgrade.test.ts | 14 +- 10 files changed, 356 insertions(+), 135 deletions(-) diff --git a/test/e2e/doctor-progress.test.ts b/test/e2e/doctor-progress.test.ts index 9f3bb7092..f318f52e8 100644 --- a/test/e2e/doctor-progress.test.ts +++ b/test/e2e/doctor-progress.test.ts @@ -20,6 +20,10 @@ import { const skip = !hasDatabase(); const describeE2E = skip ? describe.skip : describe; +if (skip) { + console.log('Skipping E2E doctor --progress-json tests (DATABASE_URL not set)'); +} + const CLI = join(import.meta.dir, '..', '..', 'src', 'cli.ts'); describeE2E('gbrain doctor --progress-json (E2E)', () => { diff --git a/test/e2e/graph-quality.test.ts b/test/e2e/graph-quality.test.ts index 8d99addb6..12a1736df9 100644 --- a/test/e2e/graph-quality.test.ts +++ b/test/e2e/graph-quality.test.ts @@ -29,9 +29,15 @@ afterAll(async () => { }); async function truncateAll() { - for (const t of ['content_chunks', 'links', 'tags', 'raw_data', 'timeline_entries', 'page_versions', 'ingest_log', 'pages']) { + for (const t of ['content_chunks', 'links', 'tags', 'raw_data', 'timeline_entries', 'page_versions', 'ingest_log', 'config', 'pages']) { await (engine as any).db.exec(`DELETE FROM ${t}`); } + // Re-seed the two config keys this file touches back to their documented + // defaults (both default to ON). This makes every test deterministic even if + // an earlier test threw before its finally restored auto_link/auto_timeline, + // and even though absent-key already resolves truthy via isAuto*Enabled. + await engine.setConfig('auto_link', 'true'); + await engine.setConfig('auto_timeline', 'true'); } function makeContext(): OperationContext { @@ -77,10 +83,12 @@ describe('E2E graph quality (v0.10.1 pipeline)', () => { await runExtract(engine, ['links', '--source', 'db']); await runExtract(engine, ['timeline', '--source', 'db']); - // Verify graph populated. + // Verify graph populated. Concrete floors derived from the seeded fixtures: + // resolvable entity refs: alice->acme, bob->acme, standup->alice, standup->bob = 4 + // timeline lines: alice(2) + bob(1) + acme(1) + standup(1) = 5 const stats = await engine.getStats(); - expect(stats.link_count).toBeGreaterThan(0); - expect(stats.timeline_entry_count).toBeGreaterThan(0); + expect(stats.link_count).toBeGreaterThanOrEqual(4); + expect(stats.timeline_entry_count).toBeGreaterThanOrEqual(5); // Verify typed link inference. const aliceLinks = await engine.getLinks('people/alice'); @@ -91,7 +99,16 @@ describe('E2E graph quality (v0.10.1 pipeline)', () => { const bobAcme = bobLinks.find(l => l.to_slug === 'companies/acme'); expect(bobAcme?.link_type).toBe('invested_in'); + // The standup meeting references both Alice and Bob as attendees. Assert the + // exact attendee edges are present and typed 'attended' (a plain .every() + // would silently pass if a meeting->company edge were misclassified or if the + // attendee edges were missing entirely). const meetingLinks = await engine.getLinks('meetings/standup'); + const attended = new Set( + meetingLinks.filter(l => l.link_type === 'attended').map(l => l.to_slug), + ); + expect(attended.has('people/alice')).toBe(true); + expect(attended.has('people/bob')).toBe(true); expect(meetingLinks.every(l => l.link_type === 'attended')).toBe(true); }); @@ -118,7 +135,9 @@ Attendees: [Alice](people/alice). Discussed [Acme](companies/acme). // The response should include auto_links results. expect((result as any).auto_links).toBeDefined(); const autoLinks = (result as any).auto_links; - expect(autoLinks.created).toBeGreaterThan(0); + // The page references exactly two seeded, resolvable targets (Alice + Acme), + // so exactly two links are created. + expect(autoLinks.created).toBe(2); expect(autoLinks.errors).toBe(0); // Verify links actually exist in DB. @@ -283,6 +302,53 @@ Mention of [Alice](people/alice). expect(paths[0].link_type).toBe('works_at'); }); + test('graph-query traversal: direction out and both, plus depth:2 multi-hop', async () => { + // Seed a 2-hop chain: alice -works_at-> acme -partnered_with-> beta. + await engine.putPage('people/alice', { type: 'person', title: 'Alice', compiled_truth: '', timeline: '' }); + await engine.putPage('companies/acme', { type: 'company', title: 'Acme', compiled_truth: '', timeline: '' }); + await engine.putPage('companies/beta', { type: 'company', title: 'Beta', compiled_truth: '', timeline: '' }); + await engine.addLink('people/alice', 'companies/acme', '', 'works_at'); + await engine.addLink('companies/acme', 'companies/beta', '', 'partnered_with'); + + // direction:'out' from alice, depth 1 -> only the first hop. + const out1 = await engine.traversePaths('people/alice', { direction: 'out', depth: 1 }); + expect(out1.length).toBe(1); + expect(out1[0].from_slug).toBe('people/alice'); + expect(out1[0].to_slug).toBe('companies/acme'); + expect(out1[0].depth).toBe(1); + + // depth:2 -> both hops, depths 1 and 2. + const out2 = await engine.traversePaths('people/alice', { direction: 'out', depth: 2 }); + const out2Edges = new Set(out2.map(p => `${p.from_slug}->${p.to_slug}@${p.depth}`)); + expect(out2Edges.has('people/alice->companies/acme@1')).toBe(true); + expect(out2Edges.has('companies/acme->companies/beta@2')).toBe(true); + expect(out2.length).toBe(2); + + // direction:'both' from acme depth 1 -> sees the inbound edge from alice AND + // the outbound edge to beta. Edges keep their natural from->to orientation. + const both = await engine.traversePaths('companies/acme', { direction: 'both', depth: 1 }); + const bothEdges = new Set(both.map(p => `${p.from_slug}->${p.to_slug}`)); + expect(bothEdges.has('people/alice->companies/acme')).toBe(true); + expect(bothEdges.has('companies/acme->companies/beta')).toBe(true); + }); + + test('graph-query cycle safety: A->B->A terminates and returns bounded results', async () => { + await engine.putPage('people/alice', { type: 'person', title: 'Alice', compiled_truth: '', timeline: '' }); + await engine.putPage('people/bob', { type: 'person', title: 'Bob', compiled_truth: '', timeline: '' }); + // Create a 2-cycle: alice -> bob -> alice. + await engine.addLink('people/alice', 'people/bob', '', 'knows'); + await engine.addLink('people/bob', 'people/alice', '', 'knows'); + + // High depth must NOT loop forever; the visited-set guard bounds the walk. + const paths = await engine.traversePaths('people/alice', { direction: 'out', depth: 100 }); + const edges = new Set(paths.map(p => `${p.from_slug}->${p.to_slug}`)); + // Both edges of the cycle are reachable exactly once. + expect(edges.has('people/alice->people/bob')).toBe(true); + expect(edges.has('people/bob->people/alice')).toBe(true); + // Bounded: there are only two edges in the graph, so no path explosion. + expect(paths.length).toBe(2); + }); + test('search backlink boost: well-connected pages rank higher', async () => { // Create 3 pages all matching a search term, but with different inbound link counts. await engine.putPage('topic/popular', { diff --git a/test/e2e/jsonb-roundtrip.test.ts b/test/e2e/jsonb-roundtrip.test.ts index 2b1b69199..9abb276f7 100644 --- a/test/e2e/jsonb-roundtrip.test.ts +++ b/test/e2e/jsonb-roundtrip.test.ts @@ -21,6 +21,10 @@ import { hasDatabase, setupDB, teardownDB, getEngine, getConn } from './helpers. const skip = !hasDatabase(); const describeE2E = skip ? describe.skip : describe; +if (skip) { + console.log('Skipping E2E JSONB roundtrip tests (DATABASE_URL not set)'); +} + describeE2E('E2E: JSONB roundtrip — v0.12.1 reliability wave', () => { beforeAll(async () => { await setupDB(); }); afterAll(async () => { await teardownDB(); }); diff --git a/test/e2e/mcp.test.ts b/test/e2e/mcp.test.ts index 530f3c036..1f985514a 100644 --- a/test/e2e/mcp.test.ts +++ b/test/e2e/mcp.test.ts @@ -56,6 +56,7 @@ describe('E2E: MCP Tool Generation', () => { expect(names).toContain('get_health'); expect(names).toContain('sync_brain'); expect(names).toContain('file_upload'); + expect(names).toContain('find_orphans'); }); test('MCP server module can be imported', async () => { diff --git a/test/e2e/mechanical.test.ts b/test/e2e/mechanical.test.ts index 8d6c52568..3b21e1e92 100644 --- a/test/e2e/mechanical.test.ts +++ b/test/e2e/mechanical.test.ts @@ -175,6 +175,15 @@ describeE2E('E2E: Search', () => { for (const [query, score] of Object.entries(scores)) { console.log(` "${query}": ${(score * 100).toFixed(0)}%`); } + + // Guard value: every known-item query must surface at least one ground-truth + // doc in the top 5. This is a deliberately loose floor (not a tuned P@5 + // threshold) — it catches a total keyword-retrieval regression without + // breaking on every scoring/fixture tweak. Without it this test asserted + // nothing and a 0%-precision result passed silently. + for (const [query, score] of Object.entries(scores)) { + expect(score).toBeGreaterThan(0); + } }); }); @@ -205,10 +214,22 @@ describeE2E('E2E: Links', () => { }, 30_000); test('traverse_graph finds connected pages', async () => { - // Links should already be added from prior test in this describe block - const graph = await callOp('traverse_graph', { slug: 'people/sarah-chen', depth: 2 }) as any; + // Self-contained: do not depend on a prior test's add_link. add_link is + // idempotent (ON CONFLICT DO NOTHING), so re-adding here is safe whether or + // not the round-trip test ran first, and the test no longer false-passes or + // false-fails based on describe-block ordering. + await callOp('add_link', { + from: 'people/sarah-chen', + to: 'companies/novamind', + link_type: 'founded', + }); + + const graph = await callOp('traverse_graph', { slug: 'people/sarah-chen', depth: 2 }) as any[]; expect(Array.isArray(graph)).toBe(true); expect(graph.length).toBeGreaterThanOrEqual(1); + // Content assertion, not just shape: the linked company must be reachable. + const reachable = graph.map((n: any) => n.slug ?? n.to_slug ?? n.to_page_slug); + expect(reachable).toContain('companies/novamind'); }); test('remove_link removes the link', async () => { @@ -469,8 +490,14 @@ describeE2E('E2E: Admin', () => { test('get_health returns valid structure', async () => { const health = await callOp('get_health') as any; expect(health).toBeDefined(); - expect(typeof health.page_count).toBe('number'); - expect(typeof health.embed_coverage).toBe('number'); + // Value bounds, not just types: page_count must match the fixture inventory + // and embed_coverage is a 0..1 fraction (src/commands/doctor.ts multiplies + // by 100 and compares to 0.9). Type-only checks let embed_coverage: -9999 + // through; these catch a genuinely broken health payload. + expect(health.page_count).toBe(16); + expect(Number.isFinite(health.embed_coverage)).toBe(true); + expect(health.embed_coverage).toBeGreaterThanOrEqual(0); + expect(health.embed_coverage).toBeLessThanOrEqual(1); }); }); @@ -488,7 +515,17 @@ describeE2E('E2E: Chunks & Resolution', () => { test('get_chunks returns chunks for imported page', async () => { const chunks = await callOp('get_chunks', { slug: 'people/sarah-chen' }) as any[]; expect(chunks.length).toBeGreaterThan(0); - expect(chunks[0].chunk_text).toBeTruthy(); + // Content + ordering, not just truthiness (a whitespace-only chunk is truthy): + // every chunk has real text and a numeric index, the indexes are + // non-decreasing in return order, and the page's own name appears somewhere. + for (const c of chunks) { + expect(typeof c.chunk_text).toBe('string'); + expect(c.chunk_text.trim().length).toBeGreaterThan(0); + expect(typeof c.chunk_index).toBe('number'); + } + const indexes = chunks.map((c: any) => c.chunk_index); + expect(indexes).toEqual([...indexes].sort((x, y) => x - y)); + expect(chunks.some((c: any) => c.chunk_text.includes('Sarah'))).toBe(true); }, 30_000); test('resolve_slugs finds partial match', async () => { @@ -662,9 +699,29 @@ describeE2E('E2E: file_list LIMIT enforcement', () => { }, 30_000); test('file_list without slug also respects LIMIT 100', async () => { - // The 150 rows from the previous test are still in the DB + // Self-sufficient: seed our own >100 rows rather than relying on the + // previous test's 150 rows surviving in the DB. A bun reorder, a focused + // `-t` run, or a failure mid-insert in the prior test would otherwise leave + // this asserting against an indeterminate row count. + const sql = getConn(); + const seedSlug = 'test-limit-noslug'; + await sql` + INSERT INTO pages (slug, title, type, compiled_truth, frontmatter) + VALUES (${seedSlug}, ${'Test Limit NoSlug'}, ${'note'}, ${'body'}, ${'{}'}::jsonb) + ON CONFLICT (source_id, slug) DO NOTHING + `; + for (let i = 0; i < 120; i++) { + await sql` + INSERT INTO files (page_slug, filename, storage_path, mime_type, size_bytes, content_hash, metadata) + VALUES (${seedSlug}, ${'nf-' + String(i).padStart(3, '0') + '.txt'}, ${seedSlug + '/nf-' + i + '.txt'}, ${'text/plain'}, ${100}, ${'nhash-' + i}, ${'{}'}::jsonb) + ON CONFLICT (storage_path) DO NOTHING + `; + } + const total = await sql`SELECT count(*)::int AS n FROM files`; + expect(Number(total[0].n)).toBeGreaterThan(100); // cap is actually exercised + const files = await callOp('file_list', {}) as any[]; - expect(files.length).toBeLessThanOrEqual(100); + expect(files.length).toBe(100); }); }); diff --git a/test/e2e/migration-flow.test.ts b/test/e2e/migration-flow.test.ts index 7ba06eebf..c0726e4be 100644 --- a/test/e2e/migration-flow.test.ts +++ b/test/e2e/migration-flow.test.ts @@ -78,6 +78,19 @@ function freshTempHome(label: string) { return dir; } +// Restore HOME/PATH to the captured originals. Called from each test's +// `finally` so a throw mid-test can never leave HOME/PATH pointed at a temp +// dir for the rest of the bun process (which would silently break unrelated +// suites that read HOME). PATH keeps the shim prepended because the +// module-level shim install is what subsequent tests in this suite rely on; +// afterAll does the final teardown to the pristine origPath. +function restoreHomePath() { + if (origHome === undefined) delete process.env.HOME; + else process.env.HOME = origHome; + if (origPath === undefined) delete process.env.PATH; + else process.env.PATH = `${fakeBinDir}:${origPath ?? ''}`; +} + beforeAll(() => { if (SKIP) { console.log('[migration-flow.e2e] DATABASE_URL not set — skipping.'); @@ -100,6 +113,15 @@ afterAll(() => { beforeEach(() => { if (SKIP) return; + // Robust restore: if a prior test threw before its own finally ran (or + // before afterAll), HOME/PATH could still point at a dead temp dir. Reset + // them to the captured originals at the start of every test so a throw in + // one test can never leak a temp HOME/PATH into sibling suites that read + // them. freshTempHome() re-points HOME per test immediately after this. + if (origHome === undefined) delete process.env.HOME; + else process.env.HOME = origHome; + if (origPath === undefined) delete process.env.PATH; + else process.env.PATH = `${fakeBinDir}:${origPath ?? ''}`; try { if (tmp) rmSync(tmp, { recursive: true, force: true }); } catch { /* best-effort */ } }); @@ -114,144 +136,160 @@ const COMMON_OPTS = { describeE2E('E2E: v0.11.0 orchestrator against live Postgres', () => { test('fresh install flow: schema → smoke → prefs → host-rewrite → completed', async () => { tmp = freshTempHome('fresh'); - const result = await v0_11_0.orchestrator(COMMON_OPTS); + try { + const result = await v0_11_0.orchestrator(COMMON_OPTS); - // Orchestrator returns a structured result (status is `complete` when - // no pending-host-work TODOs fired, `partial` otherwise). - expect(result.version).toBe('0.11.0'); - expect(['complete', 'partial']).toContain(result.status); + // Orchestrator returns a structured result (status is `complete` when + // no pending-host-work TODOs fired, `partial` otherwise). + expect(result.version).toBe('0.11.0'); + expect(['complete', 'partial']).toContain(result.status); - // Phase D: preferences.json exists with 0o600 + mode=pain_triggered. - const prefsPath = join(tmp, '.gbrain', 'preferences.json'); - expect(existsSync(prefsPath)).toBe(true); - expect(statSync(prefsPath).mode & 0o777).toBe(0o600); - const prefs = loadPreferences(); - expect(prefs.minion_mode).toBe('pain_triggered'); - expect(prefs.set_at).toBeTruthy(); - expect(prefs.set_in_version).toBeTruthy(); + // Phase D: preferences.json exists with 0o600 + mode=pain_triggered. + const prefsPath = join(tmp, '.gbrain', 'preferences.json'); + expect(existsSync(prefsPath)).toBe(true); + expect(statSync(prefsPath).mode & 0o777).toBe(0o600); + const prefs = loadPreferences(); + expect(prefs.minion_mode).toBe('pain_triggered'); + expect(prefs.set_at).toBeTruthy(); + expect(prefs.set_in_version).toBeTruthy(); - // Bug 3 (v0.14.2) — orchestrator no longer writes completed.jsonl. - // The runner (apply-migrations.ts) persists the result after the - // orchestrator returns. A direct orchestrator call in E2E leaves the - // ledger empty; the runner path is tested separately in - // test/apply-migrations.test.ts + test/migration-resume.test.ts. - const completed = loadCompletedMigrations(); - const v0110Entries = completed.filter(e => e.version === '0.11.0'); - expect(v0110Entries.length).toBe(0); + // Bug 3 (v0.14.2) — orchestrator no longer writes completed.jsonl. + // The runner (apply-migrations.ts) persists the result after the + // orchestrator returns. A direct orchestrator call in E2E leaves the + // ledger empty; the runner path is tested separately in + // test/apply-migrations.test.ts + test/migration-resume.test.ts. + const completed = loadCompletedMigrations(); + const v0110Entries = completed.filter(e => e.version === '0.11.0'); + expect(v0110Entries.length).toBe(0); - // Phase F is skipped per COMMON_OPTS — autopilot should NOT have been - // installed on this host. - expect(result.autopilot_installed).toBe(false); + // Phase F is skipped per COMMON_OPTS — autopilot should NOT have been + // installed on this host. + expect(result.autopilot_installed).toBe(false); + } finally { + restoreHomePath(); + } }, 60_000); test('idempotent rerun: second invocation is a safe no-op', async () => { tmp = freshTempHome('rerun'); - const first = await v0_11_0.orchestrator(COMMON_OPTS); - expect(['complete', 'partial']).toContain(first.status); + try { + const first = await v0_11_0.orchestrator(COMMON_OPTS); + expect(['complete', 'partial']).toContain(first.status); - const second = await v0_11_0.orchestrator(COMMON_OPTS); - expect(['complete', 'partial']).toContain(second.status); + const second = await v0_11_0.orchestrator(COMMON_OPTS); + expect(['complete', 'partial']).toContain(second.status); - // Bug 3 (v0.14.2) — orchestrator does not write completed.jsonl, so - // repeated direct invocations don't accumulate ledger entries. Assert - // the preferences state stays stable (the real idempotency signal for - // this orchestrator is "running again doesn't corrupt preferences"). - expect(loadPreferences().minion_mode).toBe('pain_triggered'); - const completed = loadCompletedMigrations(); - expect(completed.filter(e => e.version === '0.11.0').length).toBe(0); + // Bug 3 (v0.14.2) — orchestrator does not write completed.jsonl, so + // repeated direct invocations don't accumulate ledger entries. Assert + // the preferences state stays stable (the real idempotency signal for + // this orchestrator is "running again doesn't corrupt preferences"). + expect(loadPreferences().minion_mode).toBe('pain_triggered'); + const completed = loadCompletedMigrations(); + expect(completed.filter(e => e.version === '0.11.0').length).toBe(0); + } finally { + restoreHomePath(); + } }, 90_000); test('host rewrite: builtin handlers auto-rewritten, non-builtins queued as JSONL TODOs', async () => { tmp = freshTempHome('host-rewrite'); - // Fixture: AGENTS.md + cron/jobs.json with a mix of gbrain-builtin and - // non-builtin handlers. - const claudeDir = join(tmp, '.claude'); - mkdirSync(claudeDir, { recursive: true }); - writeFileSync( - join(claudeDir, 'AGENTS.md'), - '# Test AGENTS.md\n\nSome existing content referencing sessions_spawn routing.\n', - ); - mkdirSync(join(claudeDir, 'cron'), { recursive: true }); - writeFileSync( - join(claudeDir, 'cron', 'jobs.json'), - JSON.stringify({ - jobs: [ - { schedule: '*/5 * * * *', kind: 'agentTurn', skill: 'sync' }, // builtin - { schedule: '0 */30 * * *', kind: 'agentTurn', skill: 'ea-inbox-sweep' }, // non-builtin - { schedule: '*/10 * * * *', kind: 'agentTurn', skill: 'embed' }, // builtin - { schedule: '0 8 * * *', kind: 'agentTurn', skill: 'morning-briefing' }, // non-builtin - ], - }, null, 2) + '\n', - ); + try { + // Fixture: AGENTS.md + cron/jobs.json with a mix of gbrain-builtin and + // non-builtin handlers. + const claudeDir = join(tmp, '.claude'); + mkdirSync(claudeDir, { recursive: true }); + writeFileSync( + join(claudeDir, 'AGENTS.md'), + '# Test AGENTS.md\n\nSome existing content referencing sessions_spawn routing.\n', + ); + mkdirSync(join(claudeDir, 'cron'), { recursive: true }); + writeFileSync( + join(claudeDir, 'cron', 'jobs.json'), + JSON.stringify({ + jobs: [ + { schedule: '*/5 * * * *', kind: 'agentTurn', skill: 'sync' }, // builtin + { schedule: '0 */30 * * *', kind: 'agentTurn', skill: 'ea-inbox-sweep' }, // non-builtin + { schedule: '*/10 * * * *', kind: 'agentTurn', skill: 'embed' }, // builtin + { schedule: '0 8 * * *', kind: 'agentTurn', skill: 'morning-briefing' }, // non-builtin + ], + }, null, 2) + '\n', + ); - const result = await v0_11_0.orchestrator(COMMON_OPTS); + const result = await v0_11_0.orchestrator(COMMON_OPTS); - // Builtins rewritten in place; non-builtins left alone. - const cronAfter = JSON.parse(readFileSync(join(claudeDir, 'cron', 'jobs.json'), 'utf-8')); - expect(cronAfter.jobs[0].kind).toBe('shell'); // sync (builtin) - expect(cronAfter.jobs[0].cmd).toContain('gbrain jobs submit sync'); - expect(cronAfter.jobs[1].kind).toBe('agentTurn'); // ea-inbox-sweep (non-builtin) - expect(cronAfter.jobs[2].kind).toBe('shell'); // embed (builtin) - expect(cronAfter.jobs[3].kind).toBe('agentTurn'); // morning-briefing (non-builtin) + // Builtins rewritten in place; non-builtins left alone. + const cronAfter = JSON.parse(readFileSync(join(claudeDir, 'cron', 'jobs.json'), 'utf-8')); + expect(cronAfter.jobs[0].kind).toBe('shell'); // sync (builtin) + expect(cronAfter.jobs[0].cmd).toContain('gbrain jobs submit sync'); + expect(cronAfter.jobs[1].kind).toBe('agentTurn'); // ea-inbox-sweep (non-builtin) + expect(cronAfter.jobs[2].kind).toBe('shell'); // embed (builtin) + expect(cronAfter.jobs[3].kind).toBe('agentTurn'); // morning-briefing (non-builtin) - // files_rewritten counts the 2 builtin rewrites. - expect(result.files_rewritten).toBeGreaterThanOrEqual(2); + // files_rewritten counts the 2 builtin rewrites. + expect(result.files_rewritten).toBeGreaterThanOrEqual(2); - // pending_host_work counts the 2 non-builtin TODOs. - expect(result.pending_host_work).toBe(2); + // pending_host_work counts the 2 non-builtin TODOs. + expect(result.pending_host_work).toBe(2); - // Status is "partial" because non-builtin TODOs remain. - expect(result.status).toBe('partial'); + // Status is "partial" because non-builtin TODOs remain. + expect(result.status).toBe('partial'); - // AGENTS.md got the marker injected. - const agentsMdAfter = readFileSync(join(claudeDir, 'AGENTS.md'), 'utf-8'); - expect(agentsMdAfter).toContain('gbrain:subagent-routing v0.11.0'); - expect(agentsMdAfter).toContain('skills/conventions/subagent-routing.md'); + // AGENTS.md got the marker injected. + const agentsMdAfter = readFileSync(join(claudeDir, 'AGENTS.md'), 'utf-8'); + expect(agentsMdAfter).toContain('gbrain:subagent-routing v0.11.0'); + expect(agentsMdAfter).toContain('skills/conventions/subagent-routing.md'); - // JSONL TODO file written under ~/.gbrain/migrations/. - const jsonlPath = join(tmp, '.gbrain', 'migrations', 'pending-host-work.jsonl'); - expect(existsSync(jsonlPath)).toBe(true); - const lines = readFileSync(jsonlPath, 'utf-8').split('\n').filter(l => l.trim()); - expect(lines.length).toBe(2); - const todos = lines.map(l => JSON.parse(l)); - const handlers = todos.map(t => t.handler).sort(); - expect(handlers).toEqual(['ea-inbox-sweep', 'morning-briefing']); - for (const todo of todos) { - expect(todo.type).toBe('cron-handler-needs-host-registration'); - expect(todo.status).toBe('pending'); - expect(todo.manifest_path).toContain('cron/jobs.json'); + // JSONL TODO file written under ~/.gbrain/migrations/. + const jsonlPath = join(tmp, '.gbrain', 'migrations', 'pending-host-work.jsonl'); + expect(existsSync(jsonlPath)).toBe(true); + const lines = readFileSync(jsonlPath, 'utf-8').split('\n').filter(l => l.trim()); + expect(lines.length).toBe(2); + const todos = lines.map(l => JSON.parse(l)); + const handlers = todos.map(t => t.handler).sort(); + expect(handlers).toEqual(['ea-inbox-sweep', 'morning-briefing']); + for (const todo of todos) { + expect(todo.type).toBe('cron-handler-needs-host-registration'); + expect(todo.status).toBe('pending'); + expect(todo.manifest_path).toContain('cron/jobs.json'); + } + } finally { + restoreHomePath(); } }, 90_000); test('resumable: partial run → orchestrator re-run → complete', async () => { tmp = freshTempHome('resumable'); - // Simulate a stopgap-written partial entry BEFORE running the orchestrator. - mkdirSync(join(tmp, '.gbrain', 'migrations'), { recursive: true }); - writeFileSync( - join(tmp, '.gbrain', 'migrations', 'completed.jsonl'), - JSON.stringify({ - version: '0.11.0', - status: 'partial', - apply_migrations_pending: true, - mode: 'pain_triggered', - source: 'fix-v0.11.0.sh', - ts: new Date().toISOString(), - }) + '\n', - ); + try { + // Simulate a stopgap-written partial entry BEFORE running the orchestrator. + mkdirSync(join(tmp, '.gbrain', 'migrations'), { recursive: true }); + writeFileSync( + join(tmp, '.gbrain', 'migrations', 'completed.jsonl'), + JSON.stringify({ + version: '0.11.0', + status: 'partial', + apply_migrations_pending: true, + mode: 'pain_triggered', + source: 'fix-v0.11.0.sh', + ts: new Date().toISOString(), + }) + '\n', + ); - // Orchestrator re-running on a partial → should succeed (schema apply - // and smoke are idempotent; prefs are preserved from the partial - // record; host-rewrite runs its safe-skip pass). Per Bug 3 (v0.14.2), - // the orchestrator itself doesn't append to completed.jsonl — the - // runner does. The stopgap's partial entry stays unchanged here. - const result = await v0_11_0.orchestrator(COMMON_OPTS); - expect(['complete', 'partial']).toContain(result.status); + // Orchestrator re-running on a partial → should succeed (schema apply + // and smoke are idempotent; prefs are preserved from the partial + // record; host-rewrite runs its safe-skip pass). Per Bug 3 (v0.14.2), + // the orchestrator itself doesn't append to completed.jsonl — the + // runner does. The stopgap's partial entry stays unchanged here. + const result = await v0_11_0.orchestrator(COMMON_OPTS); + expect(['complete', 'partial']).toContain(result.status); - const completed = loadCompletedMigrations(); - const v0110 = completed.filter(e => e.version === '0.11.0'); - // Just the stopgap partial — orchestrator doesn't add its own entry. - expect(v0110.length).toBe(1); - expect(v0110[0].status).toBe('partial'); - expect(v0110[0].source).toBe('fix-v0.11.0.sh'); + const completed = loadCompletedMigrations(); + const v0110 = completed.filter(e => e.version === '0.11.0'); + // Just the stopgap partial — orchestrator doesn't add its own entry. + expect(v0110.length).toBe(1); + expect(v0110[0].status).toBe('partial'); + expect(v0110[0].source).toBe('fix-v0.11.0.sh'); + } finally { + restoreHomePath(); + } }, 90_000); }); diff --git a/test/e2e/minions-resilience.test.ts b/test/e2e/minions-resilience.test.ts index 21614fdd5..bd1ce0419 100644 --- a/test/e2e/minions-resilience.test.ts +++ b/test/e2e/minions-resilience.test.ts @@ -94,7 +94,7 @@ describeE2E('E2E: Minions resilience (OpenClaw real-world patterns)', () => { }, 30_000); // --- 2. Runaway handler: ignores AbortSignal, dead-lettered by handleTimeouts --- - test('runaway handler: ignores AbortSignal, handleTimeouts dead-letters in <2s', async () => { + test('runaway handler: ignores AbortSignal, handleTimeouts dead-letters it', async () => { const { a, b } = await makeEngines(); try { const queue = new MinionQueue(a); @@ -133,8 +133,14 @@ describeE2E('E2E: Minions resilience (OpenClaw real-world patterns)', () => { worker.stop(); await startP; + // Correctness gate: the job MUST be dead-lettered with the timeout reason. + // We intentionally do NOT assert a wall-clock upper bound (deadAt - started): + // on a loaded CI runner the stall/timeout sweep cadence varies, and the only + // thing that matters is that the runaway job terminates as 'dead'. The 3s poll + // deadline above is the real timeout — if the sweep is too slow, finalStatus + // stays '' and this toBe('dead') fails loudly. expect(finalStatus).toBe('dead'); - expect(deadAt - started).toBeLessThan(2000); + void deadAt; // retained for debugging; no timing assertion (flake-prone) const final = await queue.getJob(job.id); expect(final?.error_text).toMatch(/timeout exceeded/i); @@ -304,7 +310,7 @@ describeE2E('E2E: Minions resilience (OpenClaw real-world patterns)', () => { }, 60_000); // --- 5. Cascade kill under load: cancelJob aborts all live descendants --- - test('cascade kill: cancelJob on parent aborts 10 live children within 2s', async () => { + test('cascade kill: cancelJob on parent aborts 10 live children', async () => { const { a, b } = await makeEngines(); try { const queue = new MinionQueue(a); @@ -374,8 +380,12 @@ describeE2E('E2E: Minions resilience (OpenClaw real-world patterns)', () => { worker.stop(); await startP; + // Correctness gate: all 10 cooperative handlers observed the abort and the + // DB shows every descendant + root cancelled. We do NOT assert a wall-clock + // upper bound on cancelElapsed — the 3s abort poll deadline above already + // bounds the wait, and asserting a tighter time flakes on shared runners. expect(abortedChildren.size).toBe(10); - expect(cancelElapsed).toBeLessThan(3000); + void cancelElapsed; // retained for debugging; no timing assertion (flake-prone) // DB truth: every descendant + root is 'cancelled' const conn = getConn(); diff --git a/test/e2e/multi-source.test.ts b/test/e2e/multi-source.test.ts index f30bef121..5c45c6b7a 100644 --- a/test/e2e/multi-source.test.ts +++ b/test/e2e/multi-source.test.ts @@ -78,7 +78,11 @@ describeE2E('v0.18.0 multi-source — Postgres schema shape (fresh install)', () ); expect(rows.length).toBe(1); expect(rows[0].is_nullable).toBe('NO'); - expect(String(rows[0].column_default)).toContain('default'); + // Postgres renders a TEXT DEFAULT 'default' literal as `'default'::text`. + // Assert the exact stored expression rather than a loose substring so a + // drift in the schema DEFAULT (e.g. a different sentinel source id) fails + // here instead of silently passing. + expect(String(rows[0].column_default)).toBe("'default'::text"); }); test('composite UNIQUE pages(source_id, slug) replaces global UNIQUE(slug)', async () => { @@ -292,6 +296,18 @@ describeE2E('v0.18.0 multi-source — cascade delete covers every dependent row' `INSERT INTO files (source_id, page_id, filename, storage_path, content_hash) VALUES ('cascadetest', ${aliceId}, 'alice.pdf', 'cascadetest/people/alice/alice.pdf', 'fh1')`, ); + const aliceFile = await conn.unsafe( + `SELECT id FROM files WHERE source_id = 'cascadetest' AND storage_path = 'cascadetest/people/alice/alice.pdf'`, + ); + const aliceFileId = aliceFile[0].id as number; + + // file_migration_ledger row keyed on the file (FK file_id ON DELETE + // CASCADE). Removing the source cascades sources → files → ledger. + await conn.unsafe( + `INSERT INTO file_migration_ledger (file_id, storage_path_old, storage_path_new, status) + VALUES (${aliceFileId}, 'cascadetest/people/alice/alice.pdf', 'cascadetest/people/alice/alice.pdf', 'pending') + ON CONFLICT (file_id) DO NOTHING`, + ); // Sanity: everything exists expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM pages WHERE source_id = 'cascadetest'`))[0].n).toBe(2); @@ -299,6 +315,7 @@ describeE2E('v0.18.0 multi-source — cascade delete covers every dependent row' expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM timeline_entries WHERE page_id = ${aliceId}`))[0].n).toBe(1); expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM links WHERE from_page_id = ${aliceId}`))[0].n).toBe(1); expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM files WHERE source_id = 'cascadetest'`))[0].n).toBe(1); + expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM file_migration_ledger WHERE file_id = ${aliceFileId}`))[0].n).toBe(1); // Remove the source. // v0.26.5: populated sources require --confirm-destructive; --yes alone is rejected. @@ -310,6 +327,7 @@ describeE2E('v0.18.0 multi-source — cascade delete covers every dependent row' expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM timeline_entries WHERE page_id = ${aliceId}`))[0].n).toBe(0); expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM links WHERE from_page_id = ${aliceId}`))[0].n).toBe(0); expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM files WHERE source_id = 'cascadetest'`))[0].n).toBe(0); + expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM file_migration_ledger WHERE file_id = ${aliceFileId}`))[0].n).toBe(0); // The sources row itself is gone. const src = await conn.unsafe(`SELECT id FROM sources WHERE id = 'cascadetest'`); @@ -378,8 +396,10 @@ describeE2E('v0.18.0 multi-source — sync --source routes through sources table test('performSync with no sourceId falls back to global sync.repo_path', async () => { const engine = getEngine(); - // Global config is still '/some/other/default/path' from the - // previous test. Without --source, performSync uses it. + // Self-contained: set the global config this test depends on directly + // instead of inheriting the side effect of the previous test. Without + // --source, performSync must read this global key. + await engine.setConfig('sync.repo_path', '/some/other/default/path'); let err: Error | null = null; try { await performSync(engine, {}); diff --git a/test/e2e/search-quality.test.ts b/test/e2e/search-quality.test.ts index 3b2e73a77..6c61d8293 100644 --- a/test/e2e/search-quality.test.ts +++ b/test/e2e/search-quality.test.ts @@ -128,6 +128,17 @@ describe('SearchResult fields', () => { expect(r.chunk_index).toBeDefined(); expect(typeof r.chunk_index).toBe('number'); }); + + test('empty keyword query returns a defined array without throwing', async () => { + const results = await engine.searchKeyword(''); + expect(Array.isArray(results)).toBe(true); + }); + + test('zero vector search returns a defined array without throwing', async () => { + const zeroVector = new Float32Array(1536); + const results = await engine.searchVector(zeroVector); + expect(Array.isArray(results)).toBe(true); + }); }); describe('detail parameter', () => { @@ -145,9 +156,11 @@ describe('detail parameter', () => { }); test('detail=low on vector search filters to compiled_truth', async () => { - // Use a timeline-direction embedding — with detail=low, should get no results - // or only compiled_truth results + // Use a timeline-direction embedding — detail=low filters to compiled_truth. + // Vector search returns every chunk with an embedding (ordered by distance), + // so the seeded compiled_truth chunks are non-empty and ALL compiled_truth. const results = await engine.searchVector(basisEmbedding(1), { detail: 'low' }); + expect(results.length).toBeGreaterThan(0); for (const r of results) { expect(r.chunk_source).toBe('compiled_truth'); } diff --git a/test/e2e/upgrade.test.ts b/test/e2e/upgrade.test.ts index ecbe31e35..af204e489 100644 --- a/test/e2e/upgrade.test.ts +++ b/test/e2e/upgrade.test.ts @@ -73,7 +73,7 @@ describeE2E('E2E: Check-Update', () => { expect(stdout).toContain('--json'); }); - test('handles no-releases gracefully (current repo state)', async () => { + test('check-update --json contract holds regardless of real release state', async () => { const proc = Bun.spawn(['bun', 'run', 'src/cli.ts', 'check-update', '--json'], { cwd: new URL('../..', import.meta.url).pathname, stdout: 'pipe', @@ -84,8 +84,16 @@ describeE2E('E2E: Check-Update', () => { expect(exitCode).toBe(0); const output = JSON.parse(stdout); - // With no releases, should return false and an error - expect(output.update_available).toBe(false); + // Don't pin update_available to a literal value — the repo may or may not + // have a published release. Assert the JSON shape instead. + expect(typeof output.update_available).toBe('boolean'); + expect(output.current_version).toBe(VERSION); + if (output.latest_version != null) { + expect(typeof output.latest_version).toBe('string'); + } + if (output.release_url != null) { + expect(typeof output.release_url).toBe('string'); + } }); test('version comparison wiring works end-to-end', () => { From 355fbc6947f71444394109c1d39e4339273d6301 Mon Sep 17 00:00:00 2001 From: sonlndv Date: Thu, 23 Jul 2026 08:46:49 +0700 Subject: [PATCH 16/25] =?UTF-8?q?fix(doctor):=20two=20false-positive/timeo?= =?UTF-8?q?ut=20fixes=20=E2=80=94=20drift=20walk=20skips=20node=5Fmodules;?= =?UTF-8?q?=20bare-tweet=20skips=20inline-code=20+=20cited=20lines=20(#177?= =?UTF-8?q?2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(drift): skip node_modules/dist/build in multi-source drift walk The drift walker recursed into node_modules (50k+ files in RN/Astro repos), exhausting the time budget before completing, so multi_source_drift always reported 'walk hit limit/timeout' on real projects. Skip heavy non-content dirs + add a deadline check on directory descent. * fix(integrity): skip inline-code spans + [Source:] citations in bare-tweet detection Recipe/doc pages that show the CORRECT citation format inline (e.g. `Tweeted about {topic} [Source: X, @handle, date]`) were false-flagged. The fenced-code skip didn't cover inline backticks; add inline-code stripping + an explicit-citation exemption. --------- Co-authored-by: Son Le --- src/commands/integrity.ts | 11 ++++++++++- src/core/multi-source-drift.ts | 8 ++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/commands/integrity.ts b/src/commands/integrity.ts index c883c1b48..42bfdef3d 100644 --- a/src/commands/integrity.ts +++ b/src/commands/integrity.ts @@ -98,8 +98,17 @@ export function findBareTweetHits(compiledTruth: string, slug: string): BareTwee } // If the line already contains a tweet URL, it's cited — skip if (URL_NEARBY_RE.test(line)) continue; + // If the line carries an explicit source citation (e.g. + // "[Source: X, @handle, 2026-05-28]"), it's already attributed — skip. + // Catches instructional/example lines in recipe docs that demonstrate + // the CORRECT citation format. (v0.42.x) + if (/\[\s*source:/i.test(line)) continue; + // Strip inline-code spans (`...`) before matching: phrases shown as + // inline-code templates in docs are examples, not bare claims. The + // fenced-code skip above only covers ``` blocks, not inline backticks. + const lineForMatch = line.replace(/`[^`]*`/g, ''); for (const re of BARE_TWEET_PHRASES) { - const m = line.match(re); + const m = lineForMatch.match(re); if (m) { hits.push({ slug, line: i + 1, rawLine: line.trim(), phrase: m[0] }); break; // one finding per line is enough diff --git a/src/core/multi-source-drift.ts b/src/core/multi-source-drift.ts index 2064c48f4..e01f34ec4 100644 --- a/src/core/multi-source-drift.ts +++ b/src/core/multi-source-drift.ts @@ -87,6 +87,11 @@ function walkMarkdownAndMdxFiles( for (const entry of entries) { if (truncated) return; if (entry.startsWith('.')) continue; + // Skip heavy non-content dirs so the walk doesn't exhaust the time + // budget on dependency/build trees (node_modules can be 50k+ files + // with zero .md). These are never gbrain page sources. + if (entry === 'node_modules' || entry === 'dist' || entry === 'build' || + entry === '.next' || entry === 'vendor' || entry === 'target') continue; const full = join(d, entry); let isDir = false; try { @@ -95,6 +100,9 @@ function walkMarkdownAndMdxFiles( continue; } if (isDir) { + // Time check on directory descent too, so a deep dependency-free + // tree still respects the deadline even before any .md is found. + if (Date.now() >= deadlineMs) { truncated = true; return; } walk(full); continue; } From 6cf8d8d66c86d3a5421715aa2a23657d39624b85 Mon Sep 17 00:00:00 2001 From: Khaja Nazimuddin <34912639+Nazim22@users.noreply.github.com> Date: Wed, 22 Jul 2026 20:46:54 -0500 Subject: [PATCH 17/25] fix(extract): clear pre-version-bump pages in extract --stale (#1791) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `extractStaleFromDB` stamped `links_extracted_at` with each page's read `updated_at` (the D4 race-fix). But the stale predicate also flags `links_extracted_at < LINK_EXTRACTOR_VERSION_TS`. Any page last edited BEFORE the version timestamp got stamped below the threshold, so the version arm re-flagged it stale on every run — an infinite re-extract loop that never cleared the lag. Since the v112 watermark column ships with no backfill, every pre-existing page starts stale, and most pre-date the version bump. In practice this left ~97% of pages permanently stale: `extract --stale` reported "done" each run but `links_extraction_lag` never dropped. Fix: stamp `GREATEST(read updated_at, versionTs)`. Old pages lift to the threshold so the version arm clears; a real future edit still advances `updated_at` past the stamp, so the CDX-1 edited-after-stamp race protection is preserved. Adds a regression test: a page with `updated_at` before LINK_EXTRACTOR_VERSION_TS must clear after extract AND stay clear on a second run (the existing tests only used now()-dated pages, so the old-page case was uncovered). Co-authored-by: Claude Opus 4.8 --- src/commands/extract.ts | 13 ++++++++++++- test/extract-stale.test.ts | 26 ++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/commands/extract.ts b/src/commands/extract.ts index db232ec51..98176e317 100644 --- a/src/commands/extract.ts +++ b/src/commands/extract.ts @@ -1743,7 +1743,18 @@ export async function extractStaleFromDB( // `page.updated_at.toISOString()` — the JS Date is ms-truncated, so the // µs-precision DB updated_at stayed strictly greater and the page never // cleared on Postgres. Stamping the exact value makes them equal. - processedRefs.push({ slug: page.slug, source_id: page.source_id, extractedAt: page.updated_at_iso }); + // + // BUT the stamp must also clear the version-staleness clause + // (`links_extracted_at < versionTs`). A page whose updated_at predates + // versionTs would otherwise be stamped below the threshold and read as + // stale forever — a permanent re-extract loop that never clears the lag. + // GREATEST(updated_at, versionTs) preserves the race semantics (a real + // future edit advances updated_at > versionTs >= stamp → re-extracts) + // while lifting old pages to the threshold so they clear. + const stampIso = page.updated_at.getTime() >= Date.parse(versionTs) + ? page.updated_at_iso + : versionTs; + processedRefs.push({ slug: page.slug, source_id: page.source_id, extractedAt: stampIso }); } // Flush NON-swallowing (CDX-4): a throw here propagates out of the sweep so diff --git a/test/extract-stale.test.ts b/test/extract-stale.test.ts index a725e309b..f6db5ff18 100644 --- a/test/extract-stale.test.ts +++ b/test/extract-stale.test.ts @@ -209,6 +209,32 @@ describe('gbrain extract --stale', () => { expect(usRows[0]?.eq).toBe(true); }); + test('REGRESSION: page with updated_at BEFORE LINK_EXTRACTOR_VERSION_TS clears (no permanent-stale loop)', async () => { + // The v112 watermark column ships with no backfill, so every pre-existing + // page starts NULL-stale — and most pre-date the version bump. Pre-fix, + // extractStaleFromDB stamped links_extracted_at = read updated_at; for a + // page edited before LINK_EXTRACTOR_VERSION_TS the stamp landed BELOW the + // version threshold, so the version arm (links_extracted_at < versionTs) + // re-flagged it stale forever — an infinite re-extract loop that never + // cleared the lag (observed: 97% of pages stuck permanently). + await engine.putPage('people/alice', personPage('Alice')); + await engine.putPage('companies/acme', companyPage('Acme', '[Alice](people/alice) leads [Acme](companies/acme).')); + // Backdate every page to BEFORE the extractor version timestamp. + await engine.executeRaw(`UPDATE pages SET updated_at = '2020-01-01T00:00:00Z'`); + expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(2); + + await runExtract(engine, ['--stale']); + // Fixed: stamp = GREATEST(read updated_at, versionTs) → lifts old pages to + // the threshold so the version arm clears, while a real future edit still + // advances updated_at past the stamp (CDX-1 race protection preserved). + expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(0); + + // Second run must ALSO find 0 — the defining symptom of the bug was that it + // never converged. + await runExtract(engine, ['--stale']); + expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(0); + }); + test('CDX-4 (D2): a link-flush throw aborts the sweep and leaves pages UNSTAMPED', async () => { await engine.putPage('people/alice', personPage('Alice')); await engine.putPage('companies/acme', companyPage('Acme', '[Alice](people/alice) founded [Acme](companies/acme).')); From 6cf4f3122dff2a5cf559775db58fa681c766ed9e Mon Sep 17 00:00:00 2001 From: Valentin Ferriere Date: Thu, 23 Jul 2026 03:46:58 +0200 Subject: [PATCH 18/25] fix(jobs): backlinks worker defaults to check, not fix (#1853) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backlinks Minion jobs submitted with an empty payload (the sync→embed→backlinks chains enqueued after every ingestion) defaulted to action='fix', rewriting tracked brain pages with generated "Referenced in" timeline bullets on every routine run — 129 vault files polluted in one day on our production brain before we traced it. This contradicts the documented intent in src/core/cycle.ts (runPhaseBacklinks): "Maintenance cycles must not rewrite tracked brain pages with generated 'Referenced in' timeline bullets. [...] the legacy filesystem fixer remains available explicitly via `gbrain check-backlinks fix`." — the jobs-worker handler simply inverted that default. Fix: default to 'check'; 'fix' requires explicit opt-in via '{"action":"fix"}' (the documented submit shape) or `gbrain check-backlinks fix`. Both explicit paths are unchanged. Adds a structural regression test (fix-wave-structural.test.ts precedent) pinning the default, since the handler dynamically imports runBacklinksCore and walks a real repo dir — a behavioral test would require mocking that hides the regression behind a test seam. Co-authored-by: Valentin Ferriere Co-authored-by: Claude Opus 4.8 (1M context) --- src/commands/jobs.ts | 8 ++++- test/backlinks-job-default.test.ts | 47 ++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 test/backlinks-job-default.test.ts diff --git a/src/commands/jobs.ts b/src/commands/jobs.ts index 31aa24e14..c5bbe55ad 100644 --- a/src/commands/jobs.ts +++ b/src/commands/jobs.ts @@ -1664,7 +1664,13 @@ export async function registerBuiltinHandlers( worker.register('backlinks', async (job) => { const { runBacklinksCore } = await import('./backlinks.ts'); - const action: 'check' | 'fix' = job.data.action === 'check' ? 'check' : 'fix'; + // Default to 'check', not 'fix': backlinks jobs submitted with an empty + // payload (e.g. the sync→embed→backlinks chains enqueued after ingestion) + // must never rewrite tracked brain pages with generated "Referenced in" + // timeline bullets. Mirrors the documented intent in src/core/cycle.ts + // (runPhaseBacklinks). The filesystem fixer stays available explicitly + // via '{"action":"fix"}' or `gbrain check-backlinks fix`. + const action: 'check' | 'fix' = job.data.action === 'fix' ? 'fix' : 'check'; const dir = typeof job.data.dir === 'string' ? job.data.dir : (await engine.getConfig('sync.repo_path')) ?? '.'; diff --git a/test/backlinks-job-default.test.ts b/test/backlinks-job-default.test.ts new file mode 100644 index 000000000..399273438 --- /dev/null +++ b/test/backlinks-job-default.test.ts @@ -0,0 +1,47 @@ +/** + * Structural regression for the backlinks Minion handler default. + * + * Backlinks jobs submitted with an EMPTY payload (the sync→embed→backlinks + * chains enqueued after every ingestion) must run as 'check', never 'fix'. + * The pre-fix handler inverted the default (`=== 'check' ? 'check' : 'fix'`), + * so every routine post-ingestion job rewrote tracked brain pages with + * generated "Referenced in" timeline bullets — contradicting the documented + * intent in src/core/cycle.ts (runPhaseBacklinks): "Maintenance cycles must + * not rewrite tracked brain pages with generated 'Referenced in' timeline + * bullets." + * + * Source-grep is the right tool here (see fix-wave-structural.test.ts): the + * handler dynamically imports runBacklinksCore and walks a real repo dir, so + * a behavioral test would require heavy mocking that hides the regression + * behind a test seam. The rule is "this specific default must stay 'check'". + */ +import { describe, test, expect } from 'bun:test'; +import { readFileSync } from 'fs'; + +describe('backlinks Minion handler — empty payload defaults to check, not fix', () => { + const src = readFileSync('src/commands/jobs.ts', 'utf8'); + + // Isolate the backlinks register block so assertions can't accidentally + // match another handler's action parsing. + const blockMatch = src.match( + /worker\.register\('backlinks',[\s\S]*?runBacklinksCore\(\{[\s\S]*?\}\);/ + ); + + test('the backlinks handler block exists', () => { + expect(blockMatch).not.toBeNull(); + }); + + test("default action is 'check' (explicit opt-in required for 'fix')", () => { + const block = blockMatch![0]; + expect(block).toMatch( + /job\.data\.action\s*===\s*'fix'\s*\?\s*'fix'\s*:\s*'check'/ + ); + }); + + test('the inverted (fix-by-default) shape stays absent', () => { + const block = blockMatch![0]; + expect(block).not.toMatch( + /job\.data\.action\s*===\s*'check'\s*\?\s*'check'\s*:\s*'fix'/ + ); + }); +}); From bb5a66942d7a7b0992f94fc59b4710c8e30b1830 Mon Sep 17 00:00:00 2001 From: Elliot Drel <156480527+ElliotDrel@users.noreply.github.com> Date: Thu, 23 Jul 2026 03:47:03 +0200 Subject: [PATCH 19/25] fix(doctor): drop dead llm_fallback_enabled recommendation from conversation_format_coverage (#1903) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The conversation_format_coverage check recommended `gbrain config set conversation_parser.llm_fallback_enabled true`, but that config key is dead (never read) — see #1890. The recommendation is a no-op and misleads users into thinking a fallback will kick in. Drop it; keep the actionable `scan` hint. Co-authored-by: Claude Opus 4.8 --- src/commands/doctor.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index c9457d318..73dc89721 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -4963,8 +4963,7 @@ export async function buildChecks( message: `${unmatched}/${sample.length} conversation pages (${unmatchedPct.toFixed(1)}%) match NO built-in pattern. ` + `Breakdown: ${breakdown}. ` + - `Investigate: gbrain conversation-parser scan | ` + - `Enable LLM fallback (opt-in): gbrain config set conversation_parser.llm_fallback_enabled true`, + `Investigate: gbrain conversation-parser scan `, }); } else { checks.push({ From b928f40bcda494e4033f52a1887123223a047506 Mon Sep 17 00:00:00 2001 From: klampatech <73077262+klampatech@users.noreply.github.com> Date: Wed, 22 Jul 2026 20:49:05 -0500 Subject: [PATCH 20/25] fix(autopilot): export ~/.bun/bin onto PATH in cron wrapper (#2013) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The wrapper script that 'gbrain autopilot --install' writes to ~/.gbrain/autopilot-run.sh sources ~/.bashrc to inherit PATH for the exec'd gbrain binary (which has a '#!/usr/bin/env bun' shebang). The standard Debian/Ubuntu ~/.bashrc ships a non-interactive guard that returns early when bash is launched non-interactively (cron, launchd, systemd) — so PATH exports operators add to ~/.bashrc never reach the wrapper subprocess. The result: the wrapper dies silently with 'env: bun: No such file or directory', leaves a stale lockfile, and every subsequent cron tick hits the lockfile and bails. The nightly dream cycle hangs waiting on a worker that never comes back, and the wrapper's own 10-min stale-lock window is the only thing that can recover it. This bites every operator whose bashrc is the standard distro default (which is the default), and there is no warning at install time. Fix: prepend ~/.bun/bin to PATH directly in the wrapper, so it is self-contained regardless of which init file the OS loaded. Add a regression test alongside the existing zshenv/zshrc source-order test (v0.36.1.x #966) so this class of bug stays caught. --- src/commands/autopilot.ts | 9 +++++++++ test/autopilot-install.test.ts | 26 ++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/src/commands/autopilot.ts b/src/commands/autopilot.ts index 82979fb12..67a7c20bf 100644 --- a/src/commands/autopilot.ts +++ b/src/commands/autopilot.ts @@ -1186,6 +1186,15 @@ function writeWrapperScript(repoPath: string): string { # OPENAI/ANTHROPIC keys exported in zshenv reach autopilot. [ -f ~/.zshenv ] && source ~/.zshenv 2>/dev/null source ~/.zshrc 2>/dev/null || source ~/.bashrc 2>/dev/null || true +# Belt-and-suspenders PATH fix. ~/.bashrc ships with a non-interactive guard +# (\`case $- in *i*) ;; *) return;; esac\`) that exits early when launched from +# cron/systemd/launchd — so its PATH exports never reach this subprocess. +# Without bun on PATH, the exec'd gbrain (a \`#!/usr/bin/env bun\` script) fails +# silently with "env: bun: No such file or directory" and leaves a stale +# lockfile that blocks every subsequent tick. Prepending ~/.bun/bin here +# keeps the wrapper self-contained regardless of which init file the OS +# loaded. +export PATH="$HOME/.bun/bin:$PATH" exec '${safeGbrainPath}' autopilot --repo '${safeRepoPath}' `; writeFileSync(wrapperPath, wrapper, { mode: 0o755 }); diff --git a/test/autopilot-install.test.ts b/test/autopilot-install.test.ts index 023b03d7d..6355691b1 100644 --- a/test/autopilot-install.test.ts +++ b/test/autopilot-install.test.ts @@ -99,3 +99,29 @@ describe('autopilot wrapper script — env source order (v0.36.1.x #966)', () => expect(src).toMatch(/source\s+~\/\.zshrc/); }); }); + +// v0.42.x: the wrapper must export PATH with ~/.bun/bin before exec'ing +// gbrain. The exec'd gbrain has a `#!/usr/bin/env bun` shebang, and the +// standard Debian ~/.bashrc ships a non-interactive guard +// (`case $- in *i*) ;; *) return;; esac`) that exits early when cron/launchd/ +// systemd invokes bash non-interactively — so the PATH exports that +// operators put in ~/.bashrc never reach this subprocess. Without the +// explicit export the wrapper silently dies with `env: bun: No such file +// or directory`, leaves a stale lockfile, and blocks every subsequent tick +// for the 10-min stale-lock window. Regression: see Hermes `cron doctor` +// reports — this caused a 1-week nightly-cycle outage on at least one +// operator machine before being diagnosed. +describe('autopilot wrapper script — bun PATH export (v0.42.x regression)', () => { + test('wrapper exports ~/.bun/bin onto PATH before the exec', async () => { + const { readFileSync } = await import('fs'); + const src = readFileSync('src/commands/autopilot.ts', 'utf8'); + // The export line must appear inside the writeWrapperScript heredoc. + expect(src).toMatch(/export\s+PATH="\$HOME\/\.bun\/bin:\$PATH"/); + // The export must precede the exec line, otherwise env never sees it. + const exportIdx = src.search(/export\s+PATH="\$HOME\/\.bun\/bin/); + const execIdx = src.search(/exec\s+'\${safeGbrainPath}'/); + expect(exportIdx).toBeGreaterThan(0); + expect(execIdx).toBeGreaterThan(0); + expect(exportIdx).toBeLessThan(execIdx); + }); +}); From 49cf5202cb7456928d8fedeeccfbef81b0e2a034 Mon Sep 17 00:00:00 2001 From: mzkarami Date: Thu, 23 Jul 2026 03:50:49 +0200 Subject: [PATCH 21/25] fix(extract): recognize reference wikilinks (#2071) --- src/core/link-extraction.ts | 4 ++-- test/link-extraction.test.ts | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/core/link-extraction.ts b/src/core/link-extraction.ts index 6ff2f6822..8aa203480 100644 --- a/src/core/link-extraction.ts +++ b/src/core/link-extraction.ts @@ -79,11 +79,11 @@ export type LinkResolutionType = 'qualified' | 'unqualified'; /** * Directory prefix whitelist. These are the top-level slug dirs the extractor * recognizes as entity references. Upstream canonical + our extensions: - * - Gbrain canonical: people, companies, meetings, concepts, deal, civic, project, source, media, yc, projects + * - Gbrain canonical: people, companies, meetings, concepts, deal, civic, project, source, media, yc, projects, reference * - Our domain extensions: tech, finance, personal, openclaw (domain-organized wikis) * - Our entity prefix: entities (we kept some legacy entities/projects/ pages) */ -const DIR_PATTERN = '(?:people|companies|meetings|concepts|deal|civic|project|projects|source|media|yc|tech|finance|personal|openclaw|entities)'; +const DIR_PATTERN = '(?:people|companies|meetings|concepts|deal|civic|project|projects|source|media|yc|tech|finance|personal|openclaw|entities|reference)'; /** * Match `[Name](path)` markdown links pointing to entity directories. diff --git a/test/link-extraction.test.ts b/test/link-extraction.test.ts index 9a2bc4f7d..64e1db2f9 100644 --- a/test/link-extraction.test.ts +++ b/test/link-extraction.test.ts @@ -140,6 +140,17 @@ describe('extractEntityRefs', () => { expect(wikiRefs[0].needsResolution).toBe(true); }); + test('recognizes reference-page wikilinks as concrete targets', () => { + const refs = extractEntityRefs('See [[reference/mcminnville-market-data]] for source context.'); + expect(refs.length).toBe(1); + expect(refs[0]).toMatchObject({ + name: 'reference/mcminnville-market-data', + slug: 'reference/mcminnville-market-data', + dir: 'reference', + }); + expect(refs[0].needsResolution).toBeUndefined(); + }); + test('skips qualified-syntax tokens (those belong to 2a)', () => { // [[wiki:topics/ai]] looks like 2a's qualified shape — even though // it wouldn't satisfy DIR_PATTERN, 2c must not claim it either From f065eb15095b5de2957dd0e3458acf8d3a1adef5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=BA=90=E6=B3=89?= <84364275+ChenyqThu@users.noreply.github.com> Date: Wed, 22 Jul 2026 18:50:54 -0700 Subject: [PATCH 22/25] fix(cycle): extract_atoms stamps concepts so synthesize_concepts has material (#2123) (#2124) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit synthesize-concepts.ts's design comment says extract_atoms stamps a `concepts:` frontmatter field on each atom and :92 consumes ONLY that field — but the extractor never wrote it, so the atoms → concepts pipeline was dead end-to-end: every cycle reported "synthesize_concepts: skipped — no atoms with concept refs" no matter how many atoms accumulated (696 page-derived atoms / 0 with concepts on our production brain before an external backfill). Fix, all on the extractor side (no synthesize change needed): - EXTRACT_PROMPT asks for `concepts` (1-3 kebab-case TOPIC labels) with an explicit reuse-over-coinage instruction — labels must cluster, since synthesize_concepts only materializes groups of >=2. - parseAtomsResponse validates labels (kebab regex, max 3, drop invalid; empty -> undefined). - The putPage frontmatter write stamps `concepts` alongside lesson / source_quote. Tests: 4 parse cases + an end-to-end regression that goes extractor -> real frontmatter -> synthesize_concepts' OWN DB query path -> concept page. The existing tests fed synthesize via the `_atoms` seam, which is exactly how this gap survived. Validated in production ahead of this PR by stamping the same shape externally: the next synthesize_concepts run wrote 33 concept pages (T2=7/T3=26) from 60 stamped atoms, zero failures. Co-authored-by: 陈源泉 Co-authored-by: Claude Fable 5 --- src/core/cycle/extract-atoms.ts | 29 +++++++- .../extract-atoms-synthesize-concepts.test.ts | 66 +++++++++++++++++++ 2 files changed, 92 insertions(+), 3 deletions(-) diff --git a/src/core/cycle/extract-atoms.ts b/src/core/cycle/extract-atoms.ts index 4c73ea400..c71517f86 100644 --- a/src/core/cycle/extract-atoms.ts +++ b/src/core/cycle/extract-atoms.ts @@ -163,10 +163,20 @@ interface ExtractedAtom { body: string; source_quote?: string; lesson?: string; + /** + * 1-3 kebab-case topic labels for concept clustering. Consumed by + * synthesize_concepts (groups atoms by `frontmatter.concepts`; only + * labels shared by >=2 atoms materialize a concept page, so the prompt + * biases reuse-over-coinage). #2123. + */ + concepts?: string[]; virality_score?: number; emotional_register?: string; } +/** kebab-case validator for concept labels ("captive-portal", "channel-pricing"). */ +const CONCEPT_LABEL_RE = /^[a-z0-9]+(-[a-z0-9]+)*$/; + const EXTRACT_PROMPT = `You extract atomic content nuggets from a transcript. An atom is a single-source, self-contained idea that could become a tweet, @@ -177,12 +187,17 @@ quote, or short essay angle. Each atom must: Output a JSON array of atoms (1-3 per transcript, never more than 3). Each atom: {title (≤80 chars), atom_type, body (2-4 sentences), -source_quote (verbatim ≤200 chars), lesson (one sentence), virality_score -(0-100), emotional_register (one of: shocking, inspiring, funny, sobering, -practical, controversial)}. +source_quote (verbatim ≤200 chars), lesson (one sentence), concepts +(1-3 topic labels), virality_score (0-100), emotional_register (one of: +shocking, inspiring, funny, sobering, practical, controversial)}. atom_type MUST be one of: ${ATOM_TYPES.join(', ')}. +concepts are kebab-case English TOPIC labels used to cluster atoms into +concept pages (e.g. "captive-portal", "channel-pricing-strategy") — never +entity or brand names. Use the same label for the same topic across atoms; +prefer a label you already used over coining a near-synonym. + Output ONLY the JSON array, no prose.`; interface DiscoveredPage { @@ -585,6 +600,7 @@ export async function runPhaseExtractAtoms( source_hash: item.contentHash.slice(0, 16), ...(atom.source_quote && { source_quote: atom.source_quote }), ...(atom.lesson && { lesson: atom.lesson }), + ...(atom.concepts && atom.concepts.length > 0 && { concepts: atom.concepts }), ...(atom.virality_score !== undefined && { virality_score: atom.virality_score }), ...(atom.emotional_register && { emotional_register: atom.emotional_register }), extracted_at: new Date().toISOString(), @@ -721,6 +737,13 @@ export function parseAtomsResponse(raw: string): ExtractedAtom[] { body, source_quote: typeof obj.source_quote === 'string' ? obj.source_quote.slice(0, 500) : undefined, lesson: typeof obj.lesson === 'string' ? obj.lesson : undefined, + concepts: (() => { + if (!Array.isArray(obj.concepts)) return undefined; + const labels = obj.concepts + .filter((c): c is string => typeof c === 'string' && CONCEPT_LABEL_RE.test(c)) + .slice(0, 3); + return labels.length > 0 ? labels : undefined; + })(), virality_score: typeof obj.virality_score === 'number' && obj.virality_score >= 0 && diff --git a/test/cycle/extract-atoms-synthesize-concepts.test.ts b/test/cycle/extract-atoms-synthesize-concepts.test.ts index d14102495..c59e8d7b0 100644 --- a/test/cycle/extract-atoms-synthesize-concepts.test.ts +++ b/test/cycle/extract-atoms-synthesize-concepts.test.ts @@ -345,3 +345,69 @@ describe('v0.41 T6: runPhaseSynthesizeConcepts via stubbed chat', () => { expect((page[0].fm as Record).tier).toBe('T1'); }); }); + +// #2123 — extract_atoms must stamp `concepts` so synthesize_concepts has +// material. The pre-fix pipeline was broken end-to-end: the extractor +// never wrote the field, and every synthesize_concepts cycle skipped with +// "no atoms with concept refs". The earlier describe blocks feed +// synthesize via the `_atoms` seam, which is exactly how the gap survived +// — so the last test here goes extractor → REAL frontmatter → real DB +// query path → concept page. +describe('#2123: concepts label parsing', () => { + test('keeps valid kebab-case labels', () => { + const raw = `[{"title":"T","atom_type":"insight","body":"b","concepts":["captive-portal","tls-certificates"]}]`; + expect(parseAtomsResponse(raw)[0].concepts).toEqual(['captive-portal', 'tls-certificates']); + }); + + test('filters non-kebab labels, keeps the rest', () => { + const raw = `[{"title":"T","atom_type":"insight","body":"b","concepts":["Captive Portal","tp_link","UPPER","valid-label"]}]`; + expect(parseAtomsResponse(raw)[0].concepts).toEqual(['valid-label']); + }); + + test('truncates to 3 labels', () => { + const raw = `[{"title":"T","atom_type":"insight","body":"b","concepts":["a","b","c","d","e"]}]`; + expect(parseAtomsResponse(raw)[0].concepts).toEqual(['a', 'b', 'c']); + }); + + test('absent / non-array / all-invalid → undefined', () => { + expect(parseAtomsResponse(`[{"title":"T","atom_type":"insight","body":"b"}]`)[0].concepts).toBeUndefined(); + expect(parseAtomsResponse(`[{"title":"T","atom_type":"insight","body":"b","concepts":"not-an-array"}]`)[0].concepts).toBeUndefined(); + expect(parseAtomsResponse(`[{"title":"T","atom_type":"insight","body":"b","concepts":["Bad Label!"]}]`)[0].concepts).toBeUndefined(); + }); +}); + +describe('#2123: extractor stamps concepts → synthesize_concepts consumes via real DB path', () => { + test('end-to-end: atoms with shared label materialize a concept page', async () => { + const chat = stubChat(`[ + {"title":"Cert warning on guest wifi","atom_type":"insight","body":"Portal redirects to an IP-based HTTPS URL.","concepts":["captive-portal"]}, + {"title":"iPhone portal popup is flaky","atom_type":"critique","body":"CNA probe behavior differs across iOS versions.","concepts":["captive-portal"]} + ]`); + const extract = await runPhaseExtractAtoms(engine, { + _transcripts: [{ filePath: '/fake/notes.txt', content: 'content', contentHash: 'cc2123' }], + _pages: [], + _chat: chat, + }); + expect(extract.status).toBe('ok'); + expect(extract.details?.atoms_extracted).toBe(2); + + // Frontmatter really carries the label (a jsonb array, not a string). + const stamped = await engine.executeRaw<{ concepts: unknown }>( + `SELECT frontmatter->'concepts' AS concepts FROM pages WHERE type = 'atom'`, + ); + expect(stamped.length).toBe(2); + for (const row of stamped) { + const arr = typeof row.concepts === 'string' ? JSON.parse(row.concepts) : row.concepts; + expect(arr).toEqual(['captive-portal']); + } + + // NO _atoms seam: synthesize discovers the atoms through its own + // DB query — this is the path that was dead before the fix. + const synth = await runPhaseSynthesizeConcepts(engine, { _chat: stubChat('unused — T3 is deterministic') }); + expect(synth.status).toBe('ok'); + expect(synth.details?.concepts_written).toBe(1); + const concept = await engine.executeRaw<{ slug: string }>( + `SELECT slug FROM pages WHERE slug = 'concepts/captive-portal' AND type = 'concept'`, + ); + expect(concept.length).toBe(1); + }); +}); From a8a94f57424b2baa77a7ad862eccf9197751ab67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=BA=90=E6=B3=89?= <84364275+ChenyqThu@users.noreply.github.com> Date: Wed, 22 Jul 2026 18:50:59 -0700 Subject: [PATCH 23/25] fix(cycle): tombstone zero-yield pages so extract_atoms stops rediscovering them (#2144) (#2145) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Idempotency was keyed on atom rows alone — a page the LLM judges un-atomizable leaves no row, so it re-entered the discovery window every run. Two production consequences: --drain false-stopped with no_progress once the window head was mostly zero-yield pages (remaining frozen while batches report +0), and every nightly re-spent extraction budget on the same pages. Fix: - After a SUCCESSFUL chat call that parses to zero atoms, stamp the source page with frontmatter.atoms_scan_hash = contentHash16. LLM failures take the catch path and stay retryable. - discoverExtractablePages + countExtractAtomsBacklog (both variants) exclude pages whose stamp matches the CURRENT content hash prefix — content edits re-eligibilize, mirroring atom-row staleness semantics. - Drain no_progress now recounts the backlog on a zero-atom batch and only stops when it genuinely didn't shrink — tombstoning IS progress. Tests: +2 pure-loop drain cases (shrinking backlog continues / flat backlog stops) and +3 PGLite integration cases (stamp + exclusion / content-change re-eligibility / failed chat does not stamp). 29 pass / 0 fail across the two files; tsc clean. Co-authored-by: 陈源泉 Co-authored-by: Claude Fable 5 --- src/core/cycle/extract-atoms-drain.ts | 9 ++++- src/core/cycle/extract-atoms.ts | 22 +++++++++++ test/extract-atoms-drain.test.ts | 39 +++++++++++++++++++ test/extract-atoms-page-discovery.test.ts | 47 +++++++++++++++++++++++ 4 files changed, 116 insertions(+), 1 deletion(-) diff --git a/src/core/cycle/extract-atoms-drain.ts b/src/core/cycle/extract-atoms-drain.ts index 98a4bfa69..91f364d87 100644 --- a/src/core/cycle/extract-atoms-drain.ts +++ b/src/core/cycle/extract-atoms-drain.ts @@ -90,7 +90,14 @@ export async function runExtractAtomsDrain( // Stop if a batch made zero forward progress — extraction is failing or // everything left is ineligible (e.g. all skipped). Prevents a hot loop // that spends budget without draining. - if (r.extracted === 0 && r.skipped === 0) { stopped = 'no_progress'; break; } + // + // #2144: a zero-ATOM batch can still be progress — tombstoned + // zero-yield pages shrink the backlog without producing atoms. Only + // stop when the backlog count genuinely didn't move. + if (r.extracted === 0 && r.skipped === 0) { + const after = await deps.countRemaining(); + if (after === null || before === null || after >= before) { stopped = 'no_progress'; break; } + } } const remaining = await deps.countRemaining(); diff --git a/src/core/cycle/extract-atoms.ts b/src/core/cycle/extract-atoms.ts index c71517f86..82ddf7aa1 100644 --- a/src/core/cycle/extract-atoms.ts +++ b/src/core/cycle/extract-atoms.ts @@ -241,6 +241,7 @@ export async function discoverExtractablePages( AND COALESCE(p.frontmatter->>'imported_from', '') <> 'markdown-greenfield' AND COALESCE(p.frontmatter->>'dream_generated', '') <> 'true' AND length(COALESCE(p.compiled_truth, '')) >= $3 + AND COALESCE(p.frontmatter->>'atoms_scan_hash', '') <> substring(p.content_hash from 1 for 16) ${hasFilter ? "AND p.slug = ANY($5::text[])" : ''} AND NOT EXISTS ( SELECT 1 @@ -313,6 +314,7 @@ export async function countExtractAtomsBacklog( AND COALESCE(p.frontmatter->>'imported_from', '') <> 'markdown-greenfield' AND COALESCE(p.frontmatter->>'dream_generated', '') <> 'true' AND length(COALESCE(p.compiled_truth, '')) >= $3 + AND COALESCE(p.frontmatter->>'atoms_scan_hash', '') <> substring(p.content_hash from 1 for 16) AND NOT EXISTS ( SELECT 1 FROM pages atom WHERE atom.type = 'atom' AND atom.source_id = $1 @@ -326,6 +328,7 @@ export async function countExtractAtomsBacklog( AND COALESCE(p.frontmatter->>'imported_from', '') <> 'markdown-greenfield' AND COALESCE(p.frontmatter->>'dream_generated', '') <> 'true' AND length(COALESCE(p.compiled_truth, '')) >= $2 + AND COALESCE(p.frontmatter->>'atoms_scan_hash', '') <> substring(p.content_hash from 1 for 16) AND NOT EXISTS ( SELECT 1 FROM pages atom WHERE atom.type = 'atom' AND atom.source_id = p.source_id @@ -571,6 +574,25 @@ export async function runPhaseExtractAtoms( const atoms = parseAtomsResponse(result.text); if (atoms.length === 0) { + // #2144: tombstone zero-yield pages so they stop being rediscovered. + // Idempotency is keyed on atom rows — a page that yields no atoms + // leaves no row, so pre-fix it re-entered the discovery window every + // run (wedging --drain with a false no_progress and re-spending + // nightly budget on the same pages). Stamp the content hash we + // scanned; discovery skips the page only while its content is + // unchanged (edits re-eligibilize, mirroring atom-row staleness). + // Only stamped after a SUCCESSFUL chat call — LLM failures take the + // catch path below and stay retryable. + if (!opts.dryRun && item.kind === 'page') { + try { + await engine.executeRaw( + `UPDATE pages + SET frontmatter = frontmatter || jsonb_build_object('atoms_scan_hash', $1::text) + WHERE source_id = $2 AND slug = $3 AND deleted_at IS NULL`, + [item.contentHash.slice(0, 16), sourceId, item.slug], + ); + } catch { /* fail-soft: page stays rediscoverable */ } + } if (item.kind === 'transcript') transcriptsProcessed++; else pagesProcessed++; continue; diff --git a/test/extract-atoms-drain.test.ts b/test/extract-atoms-drain.test.ts index cb7dae821..ed316ecc9 100644 --- a/test/extract-atoms-drain.test.ts +++ b/test/extract-atoms-drain.test.ts @@ -134,3 +134,42 @@ describe('shared wiring helper holds the cycle lock (5A)', () => { expect(src).toContain('withRefreshingLock(engine, lockId'); }); }); + +describe('#2144: zero-yield tombstone progress semantics', () => { + it('continues when a zero-atom batch still shrinks the backlog (tombstoned pages)', async () => { + let batches = 0; + const result = await runExtractAtomsDrain( + { + withLock: passThroughLock, + // consumed: before#1=4, after#1=2 (<4 → progress), before#2=2, + // after#2=0 (<2 → progress), before#3=0 → drained; final repeats 0. + countRemaining: seq([4, 2, 2, 0, 0]), + runBatch: async () => { batches++; return { extracted: 0, skipped: 0 }; }, + now: () => 0, + }, + { windowMs: 1_000_000 }, + ); + expect(result.stopped).toBe('drained'); + expect(result.batches).toBe(2); + expect(result.extracted).toBe(0); + expect(result.remaining).toBe(0); + expect(batches).toBe(2); + }); + + it('stops no_progress when a zero-atom batch leaves the backlog flat', async () => { + let batches = 0; + const result = await runExtractAtomsDrain( + { + withLock: passThroughLock, + countRemaining: seq([5, 5]), + runBatch: async () => { batches++; return { extracted: 0, skipped: 0 }; }, + now: () => 0, + }, + { windowMs: 1_000_000 }, + ); + expect(result.stopped).toBe('no_progress'); + expect(result.batches).toBe(1); + expect(result.remaining).toBe(5); + expect(batches).toBe(1); + }); +}); diff --git a/test/extract-atoms-page-discovery.test.ts b/test/extract-atoms-page-discovery.test.ts index 7289cc52e..4a0f7b061 100644 --- a/test/extract-atoms-page-discovery.test.ts +++ b/test/extract-atoms-page-discovery.test.ts @@ -432,3 +432,50 @@ describe('v0.41.2.1: runPhaseExtractAtoms — dual-source merge + idempotency', expect(discovered.details?.atoms_extracted).toBe(1); }); }); + +describe('#2144: zero-yield tombstone', () => { + test('zero-yield page is stamped and excluded from rediscovery', async () => { + await seedPage({ slug: 'article/zero-yield', type: 'article' }); + // Successful LLM call that yields no atoms. + const result = await runPhaseExtractAtoms(engine, { _transcripts: [], _chat: stubChat('[]') }); + expect(result.details?.pages_processed).toBe(1); + expect(result.details?.atoms_extracted).toBe(0); + + // Stamp landed: atoms_scan_hash = first 16 chars of the page's content_hash. + const rows = await engine.executeRaw<{ scan: string; ch: string }>( + `SELECT frontmatter->>'atoms_scan_hash' AS scan, content_hash AS ch + FROM pages WHERE slug = 'article/zero-yield'`, + ); + expect(rows[0].scan).toBe(rows[0].ch.slice(0, 16)); + + // No longer rediscovered. + const discovered = await discoverExtractablePages(engine, 'default'); + expect(discovered.find((d) => d.slug === 'article/zero-yield')).toBeUndefined(); + }); + + test('content change re-eligibilizes a tombstoned page', async () => { + await seedPage({ slug: 'article/evolves', type: 'article' }); + await runPhaseExtractAtoms(engine, { _transcripts: [], _chat: stubChat('[]') }); + expect((await discoverExtractablePages(engine, 'default')).length).toBe(0); + + // Simulate an edit: content_hash moves while the stale stamp stays. + await engine.executeRaw( + `UPDATE pages SET content_hash = 'fresh-hash-after-edit' WHERE slug = $1 AND source_id = 'default'`, + ['article/evolves'], + ); + const rediscovered = await discoverExtractablePages(engine, 'default'); + expect(rediscovered.map((d) => d.slug)).toContain('article/evolves'); + }); + + test('failed chat does NOT stamp — page stays retryable', async () => { + await seedPage({ slug: 'article/transient-failure', type: 'article' }); + const failingChat = async (_o: ChatOpts): Promise => { throw new Error('rate limit'); }; + await runPhaseExtractAtoms(engine, { _transcripts: [], _chat: failingChat as never }); + const rows = await engine.executeRaw<{ scan: string | null }>( + `SELECT frontmatter->>'atoms_scan_hash' AS scan FROM pages WHERE slug = 'article/transient-failure'`, + ); + expect(rows[0].scan).toBeNull(); + const discovered = await discoverExtractablePages(engine, 'default'); + expect(discovered.map((d) => d.slug)).toContain('article/transient-failure'); + }); +}); From 74358329e1ddffbfa92b2da641eceffd21771fbb Mon Sep 17 00:00:00 2001 From: Brett Date: Wed, 22 Jul 2026 20:51:04 -0500 Subject: [PATCH 24/25] fix(doctor): stop claiming "Brain is at target" when the target is unreachable (#2151) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `gbrain doctor --remediation-plan` printed two consecutive lines that contradicted each other when the brain was below target AND the target was unreachable with autonomous remediation: Brain score: 45/100 → target 90 Target unreachable: max with autonomous remediation is 70/100. No remediations needed. Brain is at target. The second sentence hid the real next step (configure the prereqs that would lift `max_reachable_score`) and made the brain look healthy when it was not. Fix: gate the "Brain is at target" line on `brain_score_current >= targetScore`. When the plan is empty AND the brain is below target, the "Target unreachable" line above is already the user-facing explanation; the `Blocked checks` block below surfaces the manual gap. Extracted `renderRemediationPlanLines(plan, targetScore): string[]` as a pure helper alongside `runRemediationPlan` so the regression coverage asserts on the rendered output directly rather than mocking `console.log`. `runRemediationPlan` now joins the lines verbatim through console.log; behavior is byte-identical for every case other than the fixed contradiction. Five regression tests cover: unreachable-and-below-target (the bug case), reachable-and-at-target, exact-target, below-target-with-plan, unreachable-with-partial-plan. 38 tests across the adjacent doctor test files stay green; `bun run typecheck` clean. --- src/commands/doctor.ts | 60 ++++++++++-- test/doctor-remediation-plan-render.test.ts | 103 ++++++++++++++++++++ 2 files changed, 155 insertions(+), 8 deletions(-) create mode 100644 test/doctor-remediation-plan-render.test.ts diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index 73dc89721..e57025188 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -7777,27 +7777,71 @@ export async function runRemediationPlan( return; } - // Human output - console.log(`Brain score: ${plan.brain_score_current}/100 → target ${targetScore}`); + for (const line of renderRemediationPlanLines(plan, targetScore)) { + console.log(line); + } +} + +/** + * Human-render the remediation plan into a sequence of console lines. + * Exported for unit-test access — `runRemediationPlan` consumes it + * verbatim and only adds the JSON-mode short-circuit. + * + * Gating the "at target" line on `brain_score_current >= targetScore` + * is load-bearing: when the plan is empty AND the target is unreachable, + * the prior shape printed both "Target unreachable: …" and "Brain is at + * target" back-to-back, which contradicted itself and hid the real next + * step (manual prereq config to lift `max_reachable_score`). + */ +export function renderRemediationPlanLines( + plan: RemediationPlanShape, + targetScore: number, +): string[] { + const lines: string[] = []; + lines.push(`Brain score: ${plan.brain_score_current}/100 → target ${targetScore}`); if (plan.target_unreachable) { - console.log(`Target unreachable: max with autonomous remediation is ${plan.max_reachable_score}/100.`); + lines.push(`Target unreachable: max with autonomous remediation is ${plan.max_reachable_score}/100.`); } if (plan.plan.length === 0) { - console.log('No remediations needed. Brain is at target.'); + if (plan.brain_score_current >= targetScore) { + lines.push('No remediations needed. Brain is at target.'); + } + // When brain_score < targetScore and plan is empty, the unreachable + // line (if applicable) is the user-facing explanation; the blocked- + // checks block below surfaces the manual gap. Don't follow with a + // misleading "at target" claim. } else { - console.log(`Plan: ${plan.plan.length} step(s), est ${plan.est_total_seconds}s, est $${plan.est_total_usd_cost.toFixed(2)}`); + lines.push(`Plan: ${plan.plan.length} step(s), est ${plan.est_total_seconds}s, est $${plan.est_total_usd_cost.toFixed(2)}`); for (const step of plan.plan) { const protectedMark = step.protected ? ' [PROTECTED]' : ''; const costMark = step.est_usd_cost ? ` ($${step.est_usd_cost.toFixed(2)})` : ''; - console.log(` ${step.step}. [${step.severity}] ${step.job}${protectedMark} — ${step.rationale}${costMark}`); + lines.push(` ${step.step}. [${step.severity}] ${step.job}${protectedMark} — ${step.rationale}${costMark}`); } } if (plan.blocked.length > 0) { - console.log(`\nBlocked checks (prereq missing):`); + lines.push(`\nBlocked checks (prereq missing):`); for (const b of plan.blocked) { - console.log(` - ${b.check}: ${b.reason}`); + lines.push(` - ${b.check}: ${b.reason}`); } } + return lines; +} + +interface RemediationPlanShape { + brain_score_current: number; + target_unreachable: boolean; + max_reachable_score: number; + plan: Array<{ + step: number; + severity: string; + job: string; + protected?: boolean; + est_usd_cost?: number; + rationale: string; + }>; + est_total_seconds: number; + est_total_usd_cost: number; + blocked: Array<{ check: string; reason: string }>; } /** diff --git a/test/doctor-remediation-plan-render.test.ts b/test/doctor-remediation-plan-render.test.ts new file mode 100644 index 000000000..b73d314ff --- /dev/null +++ b/test/doctor-remediation-plan-render.test.ts @@ -0,0 +1,103 @@ +// Regression coverage for the `gbrain doctor --remediation-plan` verdict +// contradiction: when the brain was below target AND the target was +// unreachable, the human renderer printed "Target unreachable: max with +// autonomous remediation is N/100" followed immediately by "No +// remediations needed. Brain is at target." — two consecutive lines that +// contradicted each other and hid the real next step. + +import { describe, test, expect } from 'bun:test'; +import { renderRemediationPlanLines } from '../src/commands/doctor.ts'; + +type Plan = Parameters[0]; + +function planFixture(overrides: Partial): Plan { + return { + brain_score_current: 0, + target_unreachable: false, + max_reachable_score: 100, + plan: [], + est_total_seconds: 0, + est_total_usd_cost: 0, + blocked: [], + ...overrides, + }; +} + +describe('renderRemediationPlanLines', () => { + test('unreachable + brain below target — never claims "Brain is at target"', () => { + const plan = planFixture({ + brain_score_current: 45, + target_unreachable: true, + max_reachable_score: 70, + plan: [], + blocked: [{ check: 'link_density', reason: 'no enrichment keys configured' }], + }); + const text = renderRemediationPlanLines(plan, 90).join('\n'); + expect(text).toContain('Brain score: 45/100'); + expect(text).toContain('Target unreachable: max with autonomous remediation is 70/100'); + expect(text).not.toContain('Brain is at target'); + expect(text).toContain('Blocked checks'); + }); + + test('reachable, brain at or above target, no plan — emits the "at target" line', () => { + const plan = planFixture({ + brain_score_current: 95, + target_unreachable: false, + max_reachable_score: 100, + plan: [], + }); + const text = renderRemediationPlanLines(plan, 90).join('\n'); + expect(text).toContain('Brain is at target'); + expect(text).not.toContain('Target unreachable'); + }); + + test('brain at exact target with empty plan — still "at target"', () => { + const plan = planFixture({ + brain_score_current: 90, + target_unreachable: false, + plan: [], + }); + const text = renderRemediationPlanLines(plan, 90).join('\n'); + expect(text).toContain('Brain is at target'); + }); + + test('brain below target with plan steps — lists the plan, no "at target" line', () => { + const plan = planFixture({ + brain_score_current: 60, + target_unreachable: false, + max_reachable_score: 100, + est_total_seconds: 120, + est_total_usd_cost: 0.4, + plan: [ + { step: 1, severity: 'high', job: 'embed-coverage', rationale: 'missing embeddings' }, + { step: 2, severity: 'med', job: 'consolidate', rationale: 'pending entity merges', est_usd_cost: 0.4 }, + ], + }); + const lines = renderRemediationPlanLines(plan, 90); + const text = lines.join('\n'); + expect(text).toContain('Plan: 2 step(s)'); + expect(text).toContain('1. [high] embed-coverage'); + expect(text).toContain('2. [med] consolidate'); + expect(text).toContain('($0.40)'); + expect(text).not.toContain('Brain is at target'); + }); + + test('unreachable but a partial plan exists — plan prints, "at target" suppressed', () => { + const plan = planFixture({ + brain_score_current: 30, + target_unreachable: true, + max_reachable_score: 55, + est_total_seconds: 90, + est_total_usd_cost: 0.2, + plan: [ + { step: 1, severity: 'high', job: 'embed-coverage', rationale: 'reach max_reachable' }, + ], + blocked: [{ check: 'enrichment', reason: 'no provider key configured' }], + }); + const text = renderRemediationPlanLines(plan, 90).join('\n'); + expect(text).toContain('Target unreachable: max with autonomous remediation is 55/100'); + expect(text).toContain('Plan: 1 step(s)'); + expect(text).toContain('Blocked checks'); + expect(text).not.toContain('Brain is at target'); + }); +}); From 1a449bf5015e8ff33af966d9f108a0b0e81a6da9 Mon Sep 17 00:00:00 2001 From: Ryan Xie <64182766+Hippityy@users.noreply.github.com> Date: Thu, 23 Jul 2026 11:51:09 +1000 Subject: [PATCH 25/25] feat(recipes): add reranker touchpoint to OpenRouter (#2164) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OpenRouter's POST /api/v1/rerank is wire-compatible with gateway.rerank() ({query, documents, model} → {results: [{index, relevance_score}]}). This adds a recipe-only reranker touchpoint declaring four models: - cohere/rerank-v3.5 (default; $0.001/search) - cohere/rerank-4-fast ($0.002/search, 32K context) - cohere/rerank-4-pro ($0.0025/search, SOTA quality) - nvidia/llama-nemotron-rerank-vl-1b-v2:free (multimodal) Unlike embedding/chat, the reranker path strictly enforces the models allowlist — the openai-compat extended-model bypass does not apply. New rerank models must be added to this recipe before they can be called. The cost_per_1m_tokens_usd value is a pseudo-rate for the budget tracker's chars/4 heuristic — Cohere bills per-search, not per-token. At ~4K chars the estimated cost is in the right ballpark. Recipe-only change; no gateway or search-layer modifications. gateway auto-concatenates path → .../api/v1/rerank. Adds hermetic unit test (test/openrouter-reranker-recipe.test.ts) covering shape, models, default_model, path, max_payload_bytes, default_timeout_ms, and cost field. No DB, no env mutation — survives the parallel 8-shard fan-out. Verified: bun run verify (30/30 green); 285 targeted recipe+rerank+budget tests pass. Co-authored-by: Hippityy --- src/core/ai/recipes/openrouter.ts | 32 ++++++++++++++++++ test/openrouter-reranker-recipe.test.ts | 44 +++++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 test/openrouter-reranker-recipe.test.ts diff --git a/src/core/ai/recipes/openrouter.ts b/src/core/ai/recipes/openrouter.ts index 055848ac8..86782cd59 100644 --- a/src/core/ai/recipes/openrouter.ts +++ b/src/core/ai/recipes/openrouter.ts @@ -23,6 +23,14 @@ import type { Recipe } from '../types.ts'; * envelope, not every individual model's capability. When in doubt about a * specific model, check https://openrouter.ai/models. * + * Reranker: `/api/v1/rerank` proxies cross-encoder rerankers (Cohere v3.5/4-fast/4-pro + * and NVIDIA Nemotron VL). Wire shape matches `gateway.rerank()`: + * `{ query, documents, model }` → `{ results: [{ index, relevance_score }] }`. + * Unlike embedding/chat, the reranker path strictly enforces the `models` + * allowlist (no openai-compat bypass) — adding new rerank models requires a + * recipe edit. Cohere bills per-search; the `cost_per_1m_tokens_usd` value + * is a pseudo-rate for the budget tracker's `chars/4` heuristic. + * * Attribution: OpenRouter recommends `HTTP-Referer` (required for app * attribution) + `X-OpenRouter-Title` (preferred; `X-Title` kept as * back-compat alias per OR docs). Defaults to `https://gbrain.ai` / `gbrain`; @@ -99,6 +107,30 @@ export const openrouter: Recipe = { // Let upstream errors surface per-model. price_last_verified: '2026-05-20', }, + reranker: { + models: [ + 'cohere/rerank-v3.5', + 'cohere/rerank-4-fast', + 'cohere/rerank-4-pro', + 'nvidia/llama-nemotron-rerank-vl-1b-v2:free', + ], + default_model: 'cohere/rerank-v3.5', + // Cohere bills per-search, not per-token. This is a pseudo-per-1M rate + // for the budget tracker's heuristic (estimates tokens as chars/4). + // At ~4K chars/search the tracker estimates ~$0.00025 — in the right + // ballpark for the per-search bill. Patch budget-tracker.ts to honour a + // `cost_per_search_usd` field for exact accounting. + cost_per_1m_tokens_usd: 0.001, + price_last_verified: '2026-06-13', + // OpenRouter doesn't publish an explicit payload cap; 5MB matches + // ZeroEntropy's upstream limit and the gateway's pre-flight ceiling. + max_payload_bytes: 5_000_000, + // OR serves /rerank under /api/v1. base_url_default already ends in /v1, + // so gateway concatenates to …/api/v1/rerank. + path: '/rerank', + // OpenRouter rerank is fast (<200 ms p50); 5 s covers cold path safely. + default_timeout_ms: 5_000, + }, }, setup_hint: 'Get an API key at https://openrouter.ai/settings/keys, then `export OPENROUTER_API_KEY=...` and use `openrouter:/`. Optional overrides: OPENROUTER_BASE_URL (proxy), OPENROUTER_REFERER (attribution URL), OPENROUTER_TITLE (attribution name).', diff --git a/test/openrouter-reranker-recipe.test.ts b/test/openrouter-reranker-recipe.test.ts new file mode 100644 index 000000000..508b7b945 --- /dev/null +++ b/test/openrouter-reranker-recipe.test.ts @@ -0,0 +1,44 @@ +import { describe, test, expect } from 'bun:test'; +import { getRecipe } from '../src/core/ai/recipes/index.ts'; + +describe('OpenRouter recipe — reranker touchpoint', () => { + test('declares a reranker touchpoint', () => { + const r = getRecipe('openrouter'); + expect(r).toBeDefined(); + expect(r!.touchpoints.reranker).toBeDefined(); + }); + + test('models list includes all supported IDs (incl. NVIDIA :free suffix)', () => { + const m = getRecipe('openrouter')!.touchpoints.reranker!.models; + expect(m).toContain('cohere/rerank-v3.5'); + expect(m).toContain('cohere/rerank-4-fast'); + expect(m).toContain('cohere/rerank-4-pro'); + // The :free suffix must appear in full — gateway.rerank() does exact + // string matching against the allowlist (no v0.31.12 extended-model bypass + // on the rerank path), so truncating to `nvidia/.../v2` would 403. + expect(m).toContain('nvidia/llama-nemotron-rerank-vl-1b-v2:free'); + }); + + test('default_model is cohere/rerank-v3.5', () => { + const tp = getRecipe('openrouter')!.touchpoints.reranker!; + expect(tp.default_model).toBe('cohere/rerank-v3.5'); + expect(tp.models).toContain(tp.default_model); + }); + + test('path is /rerank (NOT ZeroEntropy default /models/rerank)', () => { + const tp = getRecipe('openrouter')!.touchpoints.reranker!; + expect(tp.path).toBe('/rerank'); + }); + + test('max_payload_bytes and timeout match plan', () => { + const tp = getRecipe('openrouter')!.touchpoints.reranker!; + expect(tp.max_payload_bytes).toBe(5_000_000); + expect(tp.default_timeout_ms).toBe(5_000); + }); + + test('cost_per_1m_tokens_usd is set (pseudo-rate for per-search billing)', () => { + const tp = getRecipe('openrouter')!.touchpoints.reranker!; + expect(typeof tp.cost_per_1m_tokens_usd).toBe('number'); + expect(tp.cost_per_1m_tokens_usd).toBeGreaterThan(0); + }); +});