From ea6cb025be0dfa2c62293c55e5eb04610b3bdd4a Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 16:13:51 -0700 Subject: [PATCH 01/39] fix(schema,minions): truthful bundled pack inspection + config-aware subagent auth (#3110) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(schema): make bundled pack inspection truthful (#2029) Two live bugs: - parseYamlMini had no block-scalar support, so a 'description: |' swallowed every following top-level key — the active gbrain-recommended pack loaded with 0 page types. Add parseBlockScalar for |/|-/|+ and >/>-/>+ in both mapping and sequence-sibling positions. - The bundled-pack list was hand-copied in three places (operations.ts had 2 names, mutate.ts had 3, load-active.ts had 7). New single registry src/core/schema-pack/bundled.ts carries all 7 shipped packs; every consumer derives from it. Takeover of #2029, rebased onto master (schema.ts hunks already landed). Co-authored-by: JiraiyaETH Co-Authored-By: Claude Fable 5 * fix(minions): subagent default client resolves config-stored Anthropic key (#2048) The legacy subagent path constructed a bare new Anthropic() (env-only), so launchd/MCP workers whose key lives in gbrain config (anthropic_api_key) failed auth. anthropic-key.ts now exports resolveAnthropicKey() (env first, then config; hasAnthropicKey delegates) and makeSubagentHandler passes it as apiKey. Partial takeover of #2048 — only the auth patch; the path patches were superseded by the outputRoot mechanism (#2415). Co-authored-by: JiraiyaETH Co-Authored-By: Claude Fable 5 * test(schema): point bundled-registry test at bundled.ts source of truth The bundled pack list moved from load-active.ts to bundled.ts in the truthful-inspection refactor; the T4 registry test still grepped load-active.ts source. Assert BUNDLED_PACK_NAMES directly instead. Co-Authored-By: Claude Fable 5 * fix(schema): block scalars keep '#' as literal content Inside a YAML block scalar '#' is content, not a comment; parseBlockScalar was routing lines through stripComment/isBlank, truncating descriptions like 'see issue #2029' and blanking comment-looking lines. Use the raw line inside the scalar. Adds a pinning test. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Garry Tan Co-authored-by: JiraiyaETH Co-authored-by: Claude Fable 5 --- src/core/ai/anthropic-key.ts | 16 +++++++-- src/core/minions/handlers/subagent.ts | 6 +++- src/core/operations.ts | 3 +- src/core/schema-pack/bundled.ts | 24 +++++++++++++ src/core/schema-pack/load-active.ts | 24 ++----------- src/core/schema-pack/loader.ts | 31 +++++++++++++++++ src/core/schema-pack/mutate.ts | 3 +- test/ai/anthropic-key.test.ts | 34 ++++++++++++++++++- test/lens-pack-manifests.test.ts | 12 +++---- test/operations-schema-pack.test.ts | 3 ++ test/schema-cli.test.ts | 39 ++++++++++++++++++++- test/schema-pack-loader.test.ts | 49 +++++++++++++++++++++++++++ test/schema-pack-mutate.test.ts | 5 ++- 13 files changed, 212 insertions(+), 37 deletions(-) create mode 100644 src/core/schema-pack/bundled.ts diff --git a/src/core/ai/anthropic-key.ts b/src/core/ai/anthropic-key.ts index 62fe069f9..3a63aa695 100644 --- a/src/core/ai/anthropic-key.ts +++ b/src/core/ai/anthropic-key.ts @@ -18,12 +18,22 @@ import { loadConfig } from '../config.ts'; export function hasAnthropicKey(): boolean { - if (process.env.ANTHROPIC_API_KEY) return true; + return resolveAnthropicKey() !== undefined; +} + +/** + * Resolve the actual key value: env first, then the gbrain config file. + * Callers constructing an Anthropic client directly (e.g. the legacy + * subagent path) must pass this as `apiKey` — a bare `new Anthropic()` + * only sees env, so launchd/MCP workers with config-stored keys fail. + */ +export function resolveAnthropicKey(): string | undefined { + if (process.env.ANTHROPIC_API_KEY) return process.env.ANTHROPIC_API_KEY; try { const cfg = loadConfig(); - if (cfg?.anthropic_api_key) return true; + if (cfg?.anthropic_api_key) return cfg.anthropic_api_key; } catch { // loadConfig may throw on first-run installs; treat as no key available. } - return false; + return undefined; } diff --git a/src/core/minions/handlers/subagent.ts b/src/core/minions/handlers/subagent.ts index 3852469ed..d9102dacc 100644 --- a/src/core/minions/handlers/subagent.ts +++ b/src/core/minions/handlers/subagent.ts @@ -48,6 +48,7 @@ import { logSubagentHeartbeat, } from './subagent-audit.ts'; import { resolveModel, isAnthropicProvider, TIER_DEFAULTS } from '../../model-config.ts'; +import { resolveAnthropicKey } from '../../ai/anthropic-key.ts'; import { buildSystemPrompt, DEFAULT_SUBAGENT_SYSTEM } from '../system-prompt.ts'; import { toolLoop as gatewayToolLoop } from '../../ai/gateway.ts'; import type { ChatToolDef, ChatMessage, ChatBlock, ChatResult, ToolHandler } from '../../ai/gateway.ts'; @@ -186,7 +187,10 @@ export function makeSubagentHandler(deps: SubagentDeps) { // lives at sdk.messages.create. Assigning sdk.messages directly gets the // right object; JS method-call semantics preserve `this` at the call // site (subagent.ts invokes client.create(...) with client === sdk.messages). - const makeAnthropic = deps.makeAnthropic ?? (() => new Anthropic()); + // Resolve the key env-first, then config (anthropic_api_key) — a bare + // new Anthropic() only reads env, so launchd/MCP workers whose key lives + // in the gbrain config file would fail auth (#2048). + const makeAnthropic = deps.makeAnthropic ?? (() => new Anthropic({ apiKey: resolveAnthropicKey() })); const client: MessagesClient = deps.client ?? makeAnthropic().messages; const config = deps.config ?? loadConfig() ?? ({ engine: 'postgres' } as GBrainConfig); const rateLeaseKey = deps.rateLeaseKey ?? DEFAULT_RATE_KEY; diff --git a/src/core/operations.ts b/src/core/operations.ts index d7464c190..670d066da 100644 --- a/src/core/operations.ts +++ b/src/core/operations.ts @@ -4719,7 +4719,8 @@ const list_schema_packs: Operation = { const { existsSync, readdirSync } = await import('node:fs'); const { join } = await import('node:path'); const { gbrainPath } = await import('./config.ts'); - const bundled = ['gbrain-base', 'gbrain-recommended']; + const { BUNDLED_PACK_NAMES } = await import('./schema-pack/bundled.ts'); + const bundled = [...BUNDLED_PACK_NAMES]; const installedDir = gbrainPath('schema-packs'); const installed: string[] = []; if (existsSync(installedDir)) { diff --git a/src/core/schema-pack/bundled.ts b/src/core/schema-pack/bundled.ts new file mode 100644 index 000000000..01c6bf0f0 --- /dev/null +++ b/src/core/schema-pack/bundled.ts @@ -0,0 +1,24 @@ +// Bundled schema-pack registry — single source of truth for the packs that +// ship in src/core/schema-pack/base/. Keep every bundled-pack consumer +// (CLI/MCP inspection, active-pack loading, mutation guards, upgrade +// discovery) on this one list so they cannot drift. +// +// v0.39 T8 — gbrain-base + gbrain-recommended. +// v0.41 T4 — lens packs: creator, investor, engineer, everything (meta-pack). +// v0.42 type-unification — gbrain-base-v2, the 15-type canonical successor. + +export const BUNDLED_PACK_NAMES = [ + 'gbrain-base', + 'gbrain-recommended', + 'gbrain-creator', + 'gbrain-investor', + 'gbrain-engineer', + 'gbrain-everything', + 'gbrain-base-v2', +] as const; + +export type BundledPackName = typeof BUNDLED_PACK_NAMES[number]; + +export function isBundledPackName(name: string): name is BundledPackName { + return (BUNDLED_PACK_NAMES as readonly string[]).includes(name); +} diff --git a/src/core/schema-pack/load-active.ts b/src/core/schema-pack/load-active.ts index 22d1d3658..1afde273f 100644 --- a/src/core/schema-pack/load-active.ts +++ b/src/core/schema-pack/load-active.ts @@ -37,6 +37,7 @@ import { type ResolutionInput, type ResolutionResult, } from './registry.ts'; +import { isBundledPackName } from './bundled.ts'; /** * Inputs the caller (operations.ts handler / engine query path) provides. @@ -92,28 +93,7 @@ export function _resetPackLocatorForTests(): void { * throwing UnknownPackError with a paste-ready install hint. */ function defaultPackLocator(name: string): string | null { - // v0.39 T8 — bundled packs registry. gbrain-base + gbrain-recommended - // ship in src/core/schema-pack/base/. Add a new entry here to bundle - // additional canonical packs. - // - // v0.41 T4 — lens packs join the bundle: creator (atoms + concepts + - // extract_atoms/synthesize_concepts phases), investor (theses + bet - // resolution + 3 calibration domains), engineer (gstack-learnings bridge - // + 3 calibration domains), everything (meta-pack stacking all three - // via extends + borrow_from). Each ships as a real YAML at base/.yaml. - const BUNDLED: ReadonlyArray = [ - 'gbrain-base', - 'gbrain-recommended', - 'gbrain-creator', - 'gbrain-investor', - 'gbrain-engineer', - 'gbrain-everything', - // v0.42 type-unification: 15-type canonical successor to gbrain-base. - // Ships as install default (Lane E T17) + via gbrain onboard pack - // upgrade flow (the unify-types Minion handler). - 'gbrain-base-v2', - ]; - if (BUNDLED.includes(name)) { + if (isBundledPackName(name)) { // Resolve bundled YAML relative to this source file. Works in both // direct-bun execution and bun --compile binaries. const here = dirname(fileURLToPath(import.meta.url)); diff --git a/src/core/schema-pack/loader.ts b/src/core/schema-pack/loader.ts index 2f21b5012..e418a3a9e 100644 --- a/src/core/schema-pack/loader.ts +++ b/src/core/schema-pack/loader.ts @@ -159,6 +159,29 @@ export function parseYamlMini(content: string): unknown { return parseMapping(baseIndent); } + function parseBlockScalar(parentIndent: number, folded: boolean): string { + const contentIndent = parentIndent + 2; + const out: string[] = []; + while (i < lines.length) { + const raw = lines[i]; + // Inside a block scalar everything is literal content — '#' is NOT a + // comment here, so use the raw line (no stripComment / isBlank). + if (raw.trim() === '') { + out.push(''); + i++; + continue; + } + const indent = indentOf(raw); + if (indent <= parentIndent) break; + out.push(raw.slice(Math.min(contentIndent, indent))); + i++; + } + if (folded) { + return out.join(' ').replace(/\s+$/u, ''); + } + return out.join('\n').replace(/\n+$/u, ''); + } + function parseSequence(baseIndent: number): unknown[] { const result: unknown[] = []; while (i < lines.length) { @@ -227,6 +250,10 @@ export function parseYamlMini(content: string): unknown { i++; if (rest2 === '') { map[key2] = parseBlock(nextIndent + 2); + } else if (rest2 === '|' || rest2 === '|-' || rest2 === '|+') { + map[key2] = parseBlockScalar(nextIndent, false); + } else if (rest2 === '>' || rest2 === '>-' || rest2 === '>+') { + map[key2] = parseBlockScalar(nextIndent, true); } else { map[key2] = parseScalar(rest2); } @@ -257,6 +284,10 @@ export function parseYamlMini(content: string): unknown { i++; if (rest === '') { result[key] = parseBlock(indent + 2); + } else if (rest === '|' || rest === '|-' || rest === '|+') { + result[key] = parseBlockScalar(indent, false); + } else if (rest === '>' || rest === '>-' || rest === '>+') { + result[key] = parseBlockScalar(indent, true); } else { result[key] = parseScalar(rest); } diff --git a/src/core/schema-pack/mutate.ts b/src/core/schema-pack/mutate.ts index 7a4bb63a7..eaf375e47 100644 --- a/src/core/schema-pack/mutate.ts +++ b/src/core/schema-pack/mutate.ts @@ -65,6 +65,7 @@ import { invalidateQueryCache } from './query-cache-invalidator.ts'; import { logMutationFailure, logMutationSuccess, type MutationActor, type MutationOp } from './mutate-audit.ts'; import { runFilePlaneLintRules } from './lint-rules.ts'; import { withPackLock, type PackLockOpts } from './pack-lock.ts'; +import { BUNDLED_PACK_NAMES as BUNDLED_PACK_NAME_LIST } from './bundled.ts'; import type { BrainEngine } from '../engine.ts'; export type PackFileFormat = 'json' | 'yaml'; @@ -93,7 +94,7 @@ export class SchemaPackMutationError extends Error { } } -export const BUNDLED_PACK_NAMES = new Set(['gbrain-base', 'gbrain-recommended', 'gbrain-base-v2']); +export const BUNDLED_PACK_NAMES = new Set(BUNDLED_PACK_NAME_LIST); export interface MutateResult { /** Pack name that was mutated. */ diff --git a/test/ai/anthropic-key.test.ts b/test/ai/anthropic-key.test.ts index a43ad7983..a06e9af90 100644 --- a/test/ai/anthropic-key.test.ts +++ b/test/ai/anthropic-key.test.ts @@ -10,7 +10,7 @@ import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { withEnv } from '../helpers/with-env.ts'; -import { hasAnthropicKey } from '../../src/core/ai/anthropic-key.ts'; +import { hasAnthropicKey, resolveAnthropicKey } from '../../src/core/ai/anthropic-key.ts'; const tmpDirs: string[] = []; function freshHome(withConfig?: Record): string { @@ -62,3 +62,35 @@ describe('hasAnthropicKey', () => { ); }); }); + +describe('resolveAnthropicKey (#2048 — subagent config-key auth)', () => { + test('env wins over config', async () => { + const home = freshHome({ anthropic_api_key: 'sk-from-config' }); + await withEnv( + { ANTHROPIC_API_KEY: 'sk-from-env', GBRAIN_HOME: home, DATABASE_URL: undefined, GBRAIN_DATABASE_URL: undefined }, + async () => { + expect(resolveAnthropicKey()).toBe('sk-from-env'); + }, + ); + }); + + test('config key returned when env unset', async () => { + const home = freshHome({ anthropic_api_key: 'sk-from-config' }); + await withEnv( + { ANTHROPIC_API_KEY: undefined, GBRAIN_HOME: home, DATABASE_URL: undefined, GBRAIN_DATABASE_URL: undefined }, + async () => { + expect(resolveAnthropicKey()).toBe('sk-from-config'); + }, + ); + }); + + test('neither → undefined', async () => { + const home = freshHome(); + await withEnv( + { ANTHROPIC_API_KEY: undefined, GBRAIN_HOME: home, DATABASE_URL: undefined, GBRAIN_DATABASE_URL: undefined }, + async () => { + expect(resolveAnthropicKey()).toBeUndefined(); + }, + ); + }); +}); diff --git a/test/lens-pack-manifests.test.ts b/test/lens-pack-manifests.test.ts index 0f79f90cb..78081c30e 100644 --- a/test/lens-pack-manifests.test.ts +++ b/test/lens-pack-manifests.test.ts @@ -55,13 +55,13 @@ describe('v0.41 T4: all 4 bundled lens packs parse cleanly', () => { }); describe('v0.41 T4: bundled registry includes lens packs', () => { - test('load-active.ts BUNDLED array source includes the 4 lens pack names', () => { - const loadActiveSrc = readFileSync( - join(here, '..', 'src', 'core', 'schema-pack', 'load-active.ts'), - 'utf-8', - ); + test('BUNDLED_PACK_NAMES includes the 4 lens pack names', async () => { + // The bundled list moved from load-active.ts to bundled.ts (the + // single source of truth); assert the array directly instead of + // grepping source text. + const { BUNDLED_PACK_NAMES } = await import('../src/core/schema-pack/bundled.ts'); for (const name of PACK_NAMES) { - expect(loadActiveSrc).toContain(`'${name}'`); + expect(BUNDLED_PACK_NAMES).toContain(name); } }); }); diff --git a/test/operations-schema-pack.test.ts b/test/operations-schema-pack.test.ts index 2ed47bbc0..0142a29d9 100644 --- a/test/operations-schema-pack.test.ts +++ b/test/operations-schema-pack.test.ts @@ -149,6 +149,9 @@ describe('list_schema_packs', () => { seedPack('mine'); const result = await operationsByName.list_schema_packs!.handler(ctxOf(), {}) as { bundled: string[]; installed: string[] }; expect(result.bundled).toContain('gbrain-base'); + expect(result.bundled).toContain('gbrain-recommended'); + expect(result.bundled).toContain('gbrain-base-v2'); + expect(result.bundled).toContain('gbrain-investor'); expect(result.installed).toContain('mine'); }); }); diff --git a/test/schema-cli.test.ts b/test/schema-cli.test.ts index 0019a1ffa..b74a941c0 100644 --- a/test/schema-cli.test.ts +++ b/test/schema-cli.test.ts @@ -64,11 +64,14 @@ describe('gbrain schema CLI (Phase C)', () => { expect(r.stdout + r.stderr).toMatch(/schema|active|list|show|validate|use/i); }); - test('schema list shows gbrain-base bundled', () => { + test('schema list shows all bundled packs', () => { const r = gbrain(['schema', 'list']); expect(r.code).toBe(0); expect(r.stdout).toContain('Bundled packs:'); expect(r.stdout).toContain('gbrain-base'); + expect(r.stdout).toContain('gbrain-recommended'); + expect(r.stdout).toContain('gbrain-base-v2'); + expect(r.stdout).toContain('gbrain-investor'); }); test('schema show gbrain-base prints manifest details', () => { @@ -97,6 +100,40 @@ describe('gbrain schema CLI (Phase C)', () => { expect(r.stdout).toContain('valid manifest'); }); + test('schema show/validate exposes bundled gbrain-recommended', () => { + const show = gbrain(['schema', 'show', 'gbrain-recommended']); + expect(show.code).toBe(0); + expect(show.stdout).toContain('gbrain-recommended v1.0.0'); + expect(show.stdout).toContain('Page types ('); + expect(show.stdout).toContain('meeting :: temporal'); + + const validate = gbrain(['schema', 'validate', 'gbrain-recommended']); + expect(validate.code).toBe(0); + expect(validate.stdout).toContain('valid manifest'); + }); + + test('schema show exposes bundled gbrain-base-v2 successor pack', () => { + const r = gbrain(['schema', 'show', 'gbrain-base-v2']); + expect(r.code).toBe(0); + expect(r.stdout).toContain('gbrain-base-v2 v1.0.0'); + expect(r.stdout).toContain('Page types ('); + expect(r.stdout).toContain('Link verbs (14)'); + }); + + test('schema active loads configured gbrain-recommended with real types', () => { + const home = mkdtempSync(join(tmpdir(), 'gbrain-schema-active-recommended-')); + try { + mkdirSync(join(home, '.gbrain'), { recursive: true }); + writeFileSync(join(home, '.gbrain', 'config.json'), JSON.stringify({ schema_pack: 'gbrain-recommended' }), 'utf-8'); + const r = gbrain(['schema', 'active'], { GBRAIN_HOME: home }); + expect(r.code).toBe(0); + expect(r.stdout).toContain('Active pack: gbrain-recommended'); + expect(r.stdout).not.toContain('Page types: 0'); + } finally { + rmSync(home, { recursive: true, force: true }); + } + }); + test('schema active reports default resolution', () => { const r = gbrain(['schema', 'active']); expect(r.code).toBe(0); diff --git a/test/schema-pack-loader.test.ts b/test/schema-pack-loader.test.ts index e35e5b826..34b164dc3 100644 --- a/test/schema-pack-loader.test.ts +++ b/test/schema-pack-loader.test.ts @@ -345,6 +345,34 @@ describe('YAML mini-parser', () => { expect(result.types[1].weight).toBe(2); }); + test('parses block scalar without swallowing following keys', () => { + const yaml = `name: blocky +description: | + First line. + Second line. +page_types: + - name: meeting + primitive: temporal + path_prefixes: + - meetings/ + aliases: [] + extractable: true + expert_routing: false`; + const result = parseYamlMini(yaml) as { description: string; page_types: Array> }; + expect(result.description).toBe('First line.\nSecond line.'); + expect(result.page_types).toHaveLength(1); + expect(result.page_types[0].name).toBe('meeting'); + }); + + test('block scalar keeps # as literal content, not a comment', () => { + const yaml = `description: | + See issue #2029 for context. +name: hashy`; + const result = parseYamlMini(yaml) as Record; + expect(result.description).toBe('See issue #2029 for context.'); + expect(result.name).toBe('hashy'); + }); + test('strips comments', () => { const result = parseYamlMini('# top comment\nname: value # inline comment') as Record; expect(result.name).toBe('value'); @@ -374,6 +402,27 @@ extends: null`; const pack = loadPackFromString(json, 'fixture.json'); expect(pack.name).toBe('json-pack'); }); + + test('loads block-scalar pack descriptions without losing page types', () => { + const pack = loadPackFromString(`api_version: gbrain-schema-pack-v1 +name: recommended-fixture +version: 1.0.0 +extends: gbrain-base +description: | + Operational starter pack. +page_types: + - name: meeting + primitive: temporal + path_prefixes: + - meetings/ + aliases: [] + extractable: true + expert_routing: false +link_types: []`, 'fixture.yaml'); + expect(pack.name).toBe('recommended-fixture'); + expect(pack.extends).toBe('gbrain-base'); + expect(pack.page_types.map((t) => t.name)).toContain('meeting'); + }); }); describe('ReDoS guard', () => { diff --git a/test/schema-pack-mutate.test.ts b/test/schema-pack-mutate.test.ts index 60ec9a58a..ff737cca5 100644 --- a/test/schema-pack-mutate.test.ts +++ b/test/schema-pack-mutate.test.ts @@ -103,7 +103,10 @@ describe('locateMutablePackFile — bundled guard', () => { expect(BUNDLED_PACK_NAMES.has('gbrain-recommended')).toBe(true); // v0.42 (T22): gbrain-base-v2 joins the bundled set. expect(BUNDLED_PACK_NAMES.has('gbrain-base-v2')).toBe(true); - expect(BUNDLED_PACK_NAMES.size).toBe(3); + // Derived from the single bundled registry — the lens packs (creator, + // investor, engineer, everything) are read-only too. + expect(BUNDLED_PACK_NAMES.has('gbrain-investor')).toBe(true); + expect(BUNDLED_PACK_NAMES.size).toBe(7); }); it('rejects gbrain-base-v2 with PACK_READONLY (bundled guard)', () => { From 593ba1653570466d69db6f8fa3d07b7fe2a65825 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 16:43:59 -0700 Subject: [PATCH 02/39] fix(embed): support hosted Perplexity embeddings (pplx-embed-v1-*) (#1046) (#3099) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a `perplexity` embedding recipe (OpenAI-compatible at https://api.perplexity.ai/v1, auth via PERPLEXITY_API_KEY only — never an OPENAI_API_KEY fallback) covering pplx-embed-v1-0.6b and pplx-embed-v1-4b. Perplexity's /embeddings endpoint diverges from OpenAI's wire shape in two places that break the AI SDK adapter, handled by a new perplexityCompatFetch shim (mirrors the Voyage/ZeroEntropy pattern incl. the two-layer OOM caps): - encoding_format only accepts base64_int8/base64_binary; the SDK's 'float' default is forced to 'base64_int8' outbound. - The response embedding is base64-encoded signed int8 components (natively quantized); decoded to number[] inbound so the SDK's Zod schema validates. Cosine similarity is scale-invariant, so raw int8 components rank correctly. Flexible dims (Matryoshka-style 128..native max: 1024 for 0.6b, 2560 for 4b) validate fail-loud in dims.ts + the init preflight; `dimensions` is Perplexity's native field so no wire translation is needed. default_dims is 1024 (works on a plain vector column for both models); the 4b model's full 2560 width rides the existing halfvec (>2000 dims) storage/ANN path. Pricing entries land in embedding-pricing.ts. Co-authored-by: Sinabina Co-authored-by: Claude Fable 5 --- src/core/ai/dims.ts | 41 +++++++++ src/core/ai/gateway.ts | 111 +++++++++++++++++++++++ src/core/ai/recipes/index.ts | 2 + src/core/ai/recipes/perplexity.ts | 54 ++++++++++++ src/core/embedding-dim-check.ts | 13 +++ src/core/embedding-pricing.ts | 3 + test/ai/recipe-perplexity.test.ts | 142 ++++++++++++++++++++++++++++++ 7 files changed, 366 insertions(+) create mode 100644 src/core/ai/recipes/perplexity.ts create mode 100644 test/ai/recipe-perplexity.test.ts diff --git a/src/core/ai/dims.ts b/src/core/ai/dims.ts index b88a4e175..75bf1a0ac 100644 --- a/src/core/ai/dims.ts +++ b/src/core/ai/dims.ts @@ -90,6 +90,30 @@ export function isValidOpenAITextEmbedding3Dim(modelId: string, dims: number): b return Number.isInteger(dims) && dims >= 1 && dims <= max; } +// Perplexity hosted embeddings (#1046): Matryoshka-style flexible dims, +// any integer from 128 up to the model's native size. `dimensions` is the +// native wire field (no translation needed); output encoding divergence +// (base64 int8) is handled by perplexityCompatFetch in gateway.ts. +const PERPLEXITY_EMBEDDING_MAX_DIMS: Record = { + 'pplx-embed-v1-0.6b': 1024, + 'pplx-embed-v1-4b': 2560, +}; +export const PERPLEXITY_MIN_DIMS = 128; + +export function isPerplexityEmbeddingModel(modelId: string): boolean { + return modelId in PERPLEXITY_EMBEDDING_MAX_DIMS; +} + +export function maxPerplexityEmbeddingDim(modelId: string): number | undefined { + return PERPLEXITY_EMBEDDING_MAX_DIMS[modelId]; +} + +export function isValidPerplexityDim(modelId: string, dims: number): boolean { + const max = PERPLEXITY_EMBEDDING_MAX_DIMS[modelId]; + if (max === undefined) return false; + return Number.isInteger(dims) && dims >= PERPLEXITY_MIN_DIMS && dims <= max; +} + // NVIDIA NIM hosted embedding models use asymmetric input_type values. Most // emit fixed natural dimensions, but llama-nemotron-embed-1b-v2 accepts // Matryoshka-style dimension overrides (e.g. matching an existing 1280d @@ -226,6 +250,23 @@ export function dimsProviderOptions( }, }; } + // Perplexity pplx-embed-v1-* — flexible dims via the native + // `dimensions` field. Fail-loud when the configured dim is outside + // the model's range (same rationale as the Voyage/ZE guards: the + // upstream HTTP 400 misroutes as a transient network error). + // Symmetric retrieval — inputType is never emitted. + if (isPerplexityEmbeddingModel(modelId)) { + if (!isValidPerplexityDim(modelId, dims)) { + const max = maxPerplexityEmbeddingDim(modelId)!; + throw new AIConfigError( + `Perplexity model "${modelId}" supports embedding_dimensions in ` + + `${PERPLEXITY_MIN_DIMS}..${max}, got ${dims}.`, + `Set \`embedding_dimensions\` to a value between ${PERPLEXITY_MIN_DIMS} and ${max} ` + + `in your gbrain config.`, + ); + } + return { openaiCompatible: { dimensions: dims } }; + } // NVIDIA NIM hosted embeddings are OpenAI-compatible but require // asymmetric input_type. Use passage for indexing/document-side vectors // and query for search-side vectors. Only llama-nemotron-embed-1b-v2 diff --git a/src/core/ai/gateway.ts b/src/core/ai/gateway.ts index bcf51aa54..7b814a54a 100644 --- a/src/core/ai/gateway.ts +++ b/src/core/ai/gateway.ts @@ -267,6 +267,18 @@ export class ZeroEntropyResponseTooLargeError extends Error { } } +/** Perplexity twin of the Voyage/ZE OOM caps (#1046). Int8 components are + * 1 byte each, so a real response (512 texts × 2560 dims) is ~1.3 MB — + * anything near this cap is unambiguously not legitimate. */ +const MAX_PERPLEXITY_RESPONSE_BYTES = 256 * 1024 * 1024; + +export class PerplexityResponseTooLargeError extends Error { + constructor(message: string) { + super(message); + this.name = 'PerplexityResponseTooLargeError'; + } +} + // ---- Unified auth resolution (D12=A) ---- // // Pre-v0.32, openai-compatible auth was duplicated across instantiateEmbedding, @@ -1298,6 +1310,103 @@ const openAICompatAsymmetricFetch = (async (input: RequestInfo | URL, init?: Req return fetch(typeof input === 'string' ? input : input.toString(), baseInit); }) as unknown as typeof fetch; +/** + * Perplexity compatibility shim (#1046). Perplexity's `/v1/embeddings` + * endpoint is OpenAI-shaped but diverges on two points that break the AI + * SDK's openai-compatible adapter: + * - `encoding_format` only accepts 'base64_int8' (default) or + * 'base64_binary'; the SDK sends 'float', which Perplexity rejects. + * Force 'base64_int8' on the wire. + * - The response `embedding` is a base64 string encoding SIGNED INT8 + * components (natively quantized output). The SDK schema expects + * `number[]` — decode Int8Array → number[] here. Cosine similarity is + * scale-invariant, so the raw int8 components rank correctly. + * `dimensions` is Perplexity's native field name — no translation needed + * (dims.ts emits it directly). Layer 1/Layer 2 OOM caps mirror the Voyage + * pattern. + * + * Exported for tests (behavioral coverage of the int8 decode); not part of + * the public gateway API. + */ +export const perplexityCompatFetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + // OUTBOUND: force the encoding Perplexity actually accepts. + if (init?.body && typeof init.body === 'string') { + try { + const parsed = JSON.parse(init.body); + if (parsed && typeof parsed === 'object' && parsed.encoding_format !== 'base64_int8') { + parsed.encoding_format = 'base64_int8'; + // Drop Content-Length so fetch recomputes from the new body. + const headers = new Headers(init.headers ?? {}); + headers.delete('content-length'); + init = { ...init, body: JSON.stringify(parsed), headers }; + } + } catch { + // Body wasn't JSON — pass through untouched. + } + } + + const resp = await fetch(input as any, init); + if (!resp.ok) return resp; + const ct = resp.headers.get('content-type') ?? ''; + if (!ct.toLowerCase().includes('application/json')) return resp; + + // Layer 1: Content-Length pre-check BEFORE the body is parsed. + const contentLengthHeader = resp.headers.get('content-length'); + if (contentLengthHeader) { + const len = parseInt(contentLengthHeader, 10); + if (Number.isFinite(len) && len > MAX_PERPLEXITY_RESPONSE_BYTES) { + throw new PerplexityResponseTooLargeError( + `Perplexity response Content-Length=${len} exceeds ${MAX_PERPLEXITY_RESPONSE_BYTES} bytes — ` + + `likely compromised endpoint or misconfiguration`, + ); + } + } + + // INBOUND: decode base64 int8 embeddings to number[] so the SDK's Zod + // schema validates. + try { + const json: any = await resp.clone().json(); + if (!json || typeof json !== 'object') return resp; + let modified = false; + if (Array.isArray(json.data)) { + for (const item of json.data) { + if (item && typeof item.embedding === 'string') { + // Layer 2: per-embedding cap for chunked responses that skipped + // Layer 1. base64 → bytes is the canonical 0.75 ratio. + const estDecoded = Math.ceil(item.embedding.length * 0.75); + if (estDecoded > MAX_PERPLEXITY_RESPONSE_BYTES) { + throw new PerplexityResponseTooLargeError( + `Perplexity embedding base64 exceeds ${MAX_PERPLEXITY_RESPONSE_BYTES} bytes ` + + `(estimated ${estDecoded} bytes from ${item.embedding.length} base64 chars)`, + ); + } + // base64_int8: one signed int8 per component. + const bytes = Buffer.from(item.embedding, 'base64'); + item.embedding = Array.from(new Int8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength)); + modified = true; + } + } + } + if (json.usage && typeof json.usage === 'object' && json.usage.prompt_tokens === undefined) { + json.usage.prompt_tokens = typeof json.usage.total_tokens === 'number' + ? json.usage.total_tokens + : 0; + modified = true; + } + if (!modified) return resp; + return new Response(JSON.stringify(json), { + status: resp.status, + statusText: resp.statusText, + headers: resp.headers, + }); + } catch (err) { + // OOM-cap throws MUST propagate; anything else falls back to the + // original response (same contract as voyageCompatFetch). + if (err instanceof PerplexityResponseTooLargeError) throw err; + return resp; + } +}) as unknown as typeof fetch; + async function resolveEmbeddingProvider(modelStr: string): Promise<{ model: any; recipe: Recipe; modelId: string }> { const { parsed, recipe } = resolveRecipe(modelStr); assertTouchpoint(recipe, 'embedding', parsed.modelId, getExtendedModelsForProvider(parsed.providerId)); @@ -1366,6 +1475,8 @@ function instantiateEmbedding(recipe: Recipe, modelId: string, cfg: AIGatewayCon ? zeroEntropyCompatFetch : recipe.id === 'nvidia' ? nvidiaCompatFetch + : recipe.id === 'perplexity' + ? perplexityCompatFetch : openAICompatAsymmetricFetch); const client = createOpenAICompatible({ name: recipe.id, diff --git a/src/core/ai/recipes/index.ts b/src/core/ai/recipes/index.ts index eb751ec61..e9f99f97c 100644 --- a/src/core/ai/recipes/index.ts +++ b/src/core/ai/recipes/index.ts @@ -26,6 +26,7 @@ import { llamaServerReranker } from './llama-server-reranker.ts'; import { moonshot } from './moonshot.ts'; import { mistral } from './mistral.ts'; import { nvidia } from './nvidia.ts'; +import { perplexity } from './perplexity.ts'; const ALL: Recipe[] = [ openai, @@ -48,6 +49,7 @@ const ALL: Recipe[] = [ moonshot, mistral, nvidia, + perplexity, ]; /** Map from `provider:id` key to recipe. */ diff --git a/src/core/ai/recipes/perplexity.ts b/src/core/ai/recipes/perplexity.ts new file mode 100644 index 000000000..cf7d0c415 --- /dev/null +++ b/src/core/ai/recipes/perplexity.ts @@ -0,0 +1,54 @@ +import type { Recipe } from '../types.ts'; + +/** + * Perplexity's hosted embeddings API (#1046). OpenAI-shaped at + * `POST {base}/embeddings` but diverges on the wire: + * - `encoding_format` only accepts 'base64_int8' (default) or + * 'base64_binary' — the AI SDK's 'float' default is rejected. + * - The response `embedding` is a base64 string encoding SIGNED INT8 + * components (natively quantized output), not a float array. + * Both divergences are handled by perplexityCompatFetch in gateway.ts + * (force 'base64_int8' outbound; decode Int8Array → number[] inbound). + * Cosine similarity is scale-invariant, so the raw int8 components store + * and rank correctly as floats. + * + * Models (per docs.perplexity.ai/api-reference/embeddings-post, 2026-07): + * - pplx-embed-v1-0.6b: dims 128..1024 (default 1024) + * - pplx-embed-v1-4b: dims 128..2560 (default 2560) + * The flexible-dim range validation lives in src/core/ai/dims.ts + * (PERPLEXITY_EMBEDDING_MAX_DIMS). default_dims is pinned at 1024 so both + * models work out of the box on a plain vector(N) column; users who want + * the 4b model's full 2560 width set `embedding_dimensions: 2560` and the + * existing halfvec path (dims > 2000) covers storage + ANN. + * + * Auth is PERPLEXITY_API_KEY only — deliberately NO OPENAI_API_KEY + * fallback (a Perplexity brain must never silently bill/route through + * OpenAI). If your key lives in PPLX_API_KEY, re-export it. + */ +export const perplexity: Recipe = { + id: 'perplexity', + name: 'Perplexity', + tier: 'openai-compat', + implementation: 'openai-compatible', + base_url_default: 'https://api.perplexity.ai/v1', + auth_env: { + required: ['PERPLEXITY_API_KEY'], + setup_url: 'https://www.perplexity.ai/settings/api', + }, + touchpoints: { + embedding: { + models: ['pplx-embed-v1-0.6b', 'pplx-embed-v1-4b'], + default_dims: 1024, + cost_per_1m_tokens_usd: 0.03, // pplx-embed-v1-4b; 0.6b is $0.004/M + price_last_verified: '2026-07-21', + // Perplexity enforces 120K combined tokens (and 512 texts) per + // request. Same pre-split posture as Voyage: assume a dense + // tokenizer (1 char ≈ 1 token) at 0.5 utilization; the gateway's + // recursive halving is the runtime safety net. + max_batch_tokens: 120_000, + chars_per_token: 1, + safety_factor: 0.5, + }, + }, + setup_hint: 'Get an API key at https://www.perplexity.ai/settings/api, then `export PERPLEXITY_API_KEY=...` (re-export PPLX_API_KEY if that is where your key lives).', +}; diff --git a/src/core/embedding-dim-check.ts b/src/core/embedding-dim-check.ts index f4f1e7ee8..bdeee0129 100644 --- a/src/core/embedding-dim-check.ts +++ b/src/core/embedding-dim-check.ts @@ -32,6 +32,10 @@ import { nvidiaEmbeddingDim, nvidiaEmbeddingDimOptions, supportsNvidiaEmbeddingDimension, + isPerplexityEmbeddingModel, + isValidPerplexityDim, + maxPerplexityEmbeddingDim, + PERPLEXITY_MIN_DIMS, } from './ai/dims.ts'; /** @@ -462,6 +466,15 @@ function isCustomDimValidForProvider( `(allowed: ${ZEROENTROPY_VALID_DIMS.join(', ')}).`, }; } + if (recipe.id === 'perplexity' && isPerplexityEmbeddingModel(modelId)) { + if (isValidPerplexityDim(modelId, requestedDims)) return { valid: true, error: '' }; + return { + valid: false, + error: + `Perplexity ${modelId} accepts dimensions ${PERPLEXITY_MIN_DIMS}..${maxPerplexityEmbeddingDim(modelId)}, ` + + `got ${requestedDims}.`, + }; + } if (recipe.id === 'openai' && isOpenAITextEmbedding3Model(modelId)) { if (isValidOpenAITextEmbedding3Dim(modelId, requestedDims)) return { valid: true, error: '' }; const maxDim = maxOpenAITextEmbedding3Dim(modelId); diff --git a/src/core/embedding-pricing.ts b/src/core/embedding-pricing.ts index e2b3450fb..c248abd65 100644 --- a/src/core/embedding-pricing.ts +++ b/src/core/embedding-pricing.ts @@ -44,6 +44,9 @@ export const EMBEDDING_PRICING: Record = { // Mistral (https://mistral.ai/pricing/api/, verified 2026-07-19) 'mistral:mistral-embed': { pricePerMTok: 0.10 }, 'mistral:mistral-embed-2312': { pricePerMTok: 0.10 }, + // Perplexity (https://docs.perplexity.ai/getting-started/pricing, verified 2026-07-21) + 'perplexity:pplx-embed-v1-0.6b': { pricePerMTok: 0.004 }, + 'perplexity:pplx-embed-v1-4b': { pricePerMTok: 0.03 }, }; export type PriceLookupResult = diff --git a/test/ai/recipe-perplexity.test.ts b/test/ai/recipe-perplexity.test.ts new file mode 100644 index 000000000..0b60ec335 --- /dev/null +++ b/test/ai/recipe-perplexity.test.ts @@ -0,0 +1,142 @@ +/** + * #1046 — Perplexity hosted embeddings (pplx-embed-v1-*). + * + * Covers the three seams the recipe touches: + * - recipe registration + auth (PERPLEXITY_API_KEY only, never OPENAI_API_KEY) + * - flexible-dim validation (128..native max) in dims.ts + the init + * preflight (resolveSchemaEmbeddingDim), incl. the >2000-dim 4b case + * - perplexityCompatFetch: forces encoding_format=base64_int8 outbound and + * decodes the base64 int8 embedding payload to number[] inbound + */ +import { afterEach, describe, expect, test } from 'bun:test'; +import { + dimsProviderOptions, + isPerplexityEmbeddingModel, + isValidPerplexityDim, + maxPerplexityEmbeddingDim, +} from '../../src/core/ai/dims.ts'; +import { getRecipe, RECIPES } from '../../src/core/ai/recipes/index.ts'; +import { perplexity } from '../../src/core/ai/recipes/perplexity.ts'; +import { defaultResolveAuth, perplexityCompatFetch } from '../../src/core/ai/gateway.ts'; +import { AIConfigError } from '../../src/core/ai/errors.ts'; +import { resolveSchemaEmbeddingDim } from '../../src/core/embedding-dim-check.ts'; +import { lookupEmbeddingPrice } from '../../src/core/embedding-pricing.ts'; + +describe('recipe: perplexity', () => { + test('registered as an OpenAI-compatible embedding provider', () => { + expect(RECIPES.has('perplexity')).toBe(true); + expect(getRecipe('perplexity')).toBe(perplexity); + expect(perplexity.tier).toBe('openai-compat'); + expect(perplexity.implementation).toBe('openai-compatible'); + expect(perplexity.base_url_default).toBe('https://api.perplexity.ai/v1'); + const e = perplexity.touchpoints.embedding!; + expect(e.models).toEqual(['pplx-embed-v1-0.6b', 'pplx-embed-v1-4b']); + expect(e.default_dims).toBe(1024); + expect(e.max_batch_tokens).toBe(120_000); + }); + + test('auth is PERPLEXITY_API_KEY bearer — no OPENAI_API_KEY fallback', () => { + expect(perplexity.resolveAuth).toBeUndefined(); + expect(perplexity.auth_env?.required).toEqual(['PERPLEXITY_API_KEY']); + expect(defaultResolveAuth(perplexity, { PERPLEXITY_API_KEY: 'fake-pplx' }, 'embedding')).toEqual({ + headerName: 'Authorization', + token: 'Bearer fake-pplx', + }); + // An OPENAI_API_KEY in the env must NOT satisfy Perplexity auth. + expect(() => defaultResolveAuth(perplexity, { OPENAI_API_KEY: 'sk-test' }, 'embedding')).toThrow(AIConfigError); + }); + + test('dims: 128..native-max range per model', () => { + expect(isPerplexityEmbeddingModel('pplx-embed-v1-4b')).toBe(true); + expect(maxPerplexityEmbeddingDim('pplx-embed-v1-4b')).toBe(2560); + expect(maxPerplexityEmbeddingDim('pplx-embed-v1-0.6b')).toBe(1024); + expect(isValidPerplexityDim('pplx-embed-v1-4b', 2560)).toBe(true); + expect(isValidPerplexityDim('pplx-embed-v1-4b', 128)).toBe(true); + expect(isValidPerplexityDim('pplx-embed-v1-4b', 64)).toBe(false); + expect(isValidPerplexityDim('pplx-embed-v1-0.6b', 2560)).toBe(false); + }); + + test('dimsProviderOptions emits native `dimensions`, fails loud out of range', () => { + expect(dimsProviderOptions('openai-compatible', 'pplx-embed-v1-4b', 2560)).toEqual({ + openaiCompatible: { dimensions: 2560 }, + }); + // Symmetric provider — inputType never emitted. + expect(dimsProviderOptions('openai-compatible', 'pplx-embed-v1-4b', 1024, 'query')).toEqual({ + openaiCompatible: { dimensions: 1024 }, + }); + expect(() => dimsProviderOptions('openai-compatible', 'pplx-embed-v1-0.6b', 2560)).toThrow(AIConfigError); + }); + + test('init preflight accepts the 4b model at its native 2560 dims (halfvec territory)', () => { + const res = resolveSchemaEmbeddingDim({ + embedding_model: 'perplexity:pplx-embed-v1-4b', + embedding_dimensions: 2560, + }); + expect(res).toEqual({ + ok: true, + dim: 2560, + model: 'perplexity:pplx-embed-v1-4b', + provider: 'perplexity', + recipeDefault: 1024, + }); + const bad = resolveSchemaEmbeddingDim({ + embedding_model: 'perplexity:pplx-embed-v1-4b', + embedding_dimensions: 4096, + }); + expect(bad.ok).toBe(false); + }); + + test('embedding pricing table knows both models', () => { + expect(lookupEmbeddingPrice('perplexity:pplx-embed-v1-4b')).toMatchObject({ kind: 'known', pricePerMTok: 0.03 }); + expect(lookupEmbeddingPrice('perplexity:pplx-embed-v1-0.6b')).toMatchObject({ kind: 'known', pricePerMTok: 0.004 }); + }); +}); + +describe('perplexityCompatFetch — int8 wire shim', () => { + const realFetch = globalThis.fetch; + afterEach(() => { + globalThis.fetch = realFetch; + }); + + test('forces encoding_format=base64_int8 outbound and decodes int8 base64 inbound', async () => { + const int8 = new Int8Array([3, -7, 127, -128]); + const b64 = Buffer.from(int8.buffer).toString('base64'); + let sentBody: any; + globalThis.fetch = (async (_input: any, init?: RequestInit) => { + sentBody = JSON.parse(init!.body as string); + return new Response( + JSON.stringify({ + object: 'list', + model: 'pplx-embed-v1-4b', + data: [{ object: 'embedding', index: 0, embedding: b64 }], + usage: { prompt_tokens: 4, total_tokens: 4 }, + }), + { status: 200, headers: { 'content-type': 'application/json' } }, + ); + }) as any; + + const resp = await (perplexityCompatFetch as any)('https://api.perplexity.ai/v1/embeddings', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + // The AI SDK sends encoding_format:'float' — Perplexity rejects it. + body: JSON.stringify({ model: 'pplx-embed-v1-4b', input: ['hi'], encoding_format: 'float', dimensions: 4 }), + }); + + expect(sentBody.encoding_format).toBe('base64_int8'); + expect(sentBody.dimensions).toBe(4); // native field, untouched + const json = await resp.json(); + expect(json.data[0].embedding).toEqual([3, -7, 127, -128]); + expect(json.usage.prompt_tokens).toBe(4); + }); + + test('non-JSON and error responses pass through untouched', async () => { + globalThis.fetch = (async () => + new Response('nope', { status: 401, headers: { 'content-type': 'text/plain' } })) as any; + const resp = await (perplexityCompatFetch as any)('https://api.perplexity.ai/v1/embeddings', { + method: 'POST', + body: JSON.stringify({ model: 'pplx-embed-v1-4b', input: ['hi'] }), + }); + expect(resp.status).toBe(401); + expect(await resp.text()).toBe('nope'); + }); +}); From 5e8816e7d641840d1a99cc57cf2ec9659808d824 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 16:45:27 -0700 Subject: [PATCH 03/39] fix(links): resolve [[wikilink]] + slug-path frontmatter values; frontmatter-fresh incremental extract (#3087) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(links): resolve [[wikilink]] + slug-path frontmatter values; keep frontmatter links fresh on the incremental cycle Takeover/rebase of two community PRs: PR #1983 — frontmatter link fields never resolved Obsidian-style values: - makeResolver step 1's strict slug regex rejected digit-leading folders (90-people/nicolai) and nested paths (a/b/c); broadened to any slug-shaped value with an EXACT getPage match only (no fuzzy, no false positives). - extractFrontmatterLinks resolved "[[dir/slug]]" verbatim; new anchored unwrapWikilink() strips wholly-wrapped [[...]] (and |alias/#heading/^block) before resolution. Bare values pass through unchanged. - Same broadened slug-shape applied to the fs-path synthetic resolver in extractLinksFromFile (exact Set membership guards it), so the fs frontmatter path resolves PARA-numbered slugs too. PR #2434 — the cycle's incremental extract (extractForSlugs) extracted body links only, so externally-edited YAML (sources:/related:) edges drifted stale. Adds an includeFrontmatter opt (threaded as a param after sourceId, which master added in #1747/#1503 after the PR was cut), gated by the new config key autopilot.incremental_extract_include_frontmatter (default off, preserves body-only behavior). Tests: unwrapWikilink unit coverage, broadened-resolver + end-to-end frontmatter cases in test/link-extraction.test.ts; fs-resolver digit-leading case in test/extract.test.ts; incremental gate off/on cases in test/extract-incremental.test.ts. Co-authored-by: spiky02plateau Co-Authored-By: Claude Fable 5 * fix(cycle): honor DB-plane config for incremental_extract_include_frontmatter The gate read loadConfig() (file/env plane) only, but the documented enable command — gbrain config set autopilot.incremental_extract_include_frontmatter true — writes the DB plane via engine.setConfig, so the feature could never be turned on the documented way (silent no-op, #2120 class). Now the file plane wins when the key is present there; otherwise the DB plane is consulted, matching the autopilot.auto_drain.* read pattern. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Sinabina Co-authored-by: spiky02plateau Co-authored-by: Claude Fable 5 Co-authored-by: Garry Tan --- src/commands/extract.ts | 25 ++++- src/core/config.ts | 12 +++ src/core/cycle.ts | 16 +++ src/core/link-extraction.ts | 39 ++++++- test/extract-incremental.test.ts | 34 ++++++ test/extract.test.ts | 12 +++ test/link-extraction.test.ts | 173 +++++++++++++++++++++++++++++++ 7 files changed, 305 insertions(+), 6 deletions(-) diff --git a/src/commands/extract.ts b/src/commands/extract.ts index b2aabc5fd..75645ca58 100644 --- a/src/commands/extract.ts +++ b/src/commands/extract.ts @@ -433,7 +433,10 @@ export async function extractLinksFromFile( async resolve(name: string, dirHint?: string | string[]): Promise { if (!name) return null; const trimmed = name.trim(); - if (/^[a-z][a-z0-9-]*\/[a-z0-9][a-z0-9-]*$/.test(trimmed) && allSlugs.has(trimmed)) { + // Same broadened slug-shape as makeResolver step 1: accepts + // digit-leading folders (`90-people/nicolai`) and nested paths. + // Exact Set membership guards it — no false positives. + if (/\//.test(trimmed) && /^[a-z0-9][a-z0-9/_-]*$/.test(trimmed) && allSlugs.has(trimmed)) { return trimmed; } const hints = Array.isArray(dirHint) ? dirHint : (dirHint ? [dirHint] : []); @@ -582,6 +585,17 @@ export interface ExtractOpts { * before (single-'default'-source brains unaffected). */ sourceId?: string; + /** + * v0.42 — also extract frontmatter links on the incremental (slugs) path. + * `extractForSlugs` extracts BODY links only by default; set this true to also + * parse each changed page's frontmatter so `sources:`/`related:` edges stay fresh + * when YAML is edited externally and synced in. Applied PER changed page, so the + * incremental walk stays bounded (no switch to a full DB scan). Only honored on + * the incremental path (`slugs` defined); the full-walk path already covers + * frontmatter via its own dispatch. Gated upstream by the config key + * `autopilot.incremental_extract_include_frontmatter` (default off). + */ + includeFrontmatter?: boolean; } /** @@ -620,7 +634,7 @@ export async function runExtractCore(engine: BrainEngine, opts: ExtractOpts): Pr // Nothing changed — skip entirely. return result; } - const r = await extractForSlugs(engine, opts.dir, opts.slugs, opts.mode, dryRun, jsonMode, workers, opts.signal, opts.sourceId); + const r = await extractForSlugs(engine, opts.dir, opts.slugs, opts.mode, dryRun, jsonMode, workers, opts.signal, opts.sourceId, opts.includeFrontmatter); result.links_created = r.links_created; result.timeline_entries_created = r.timeline_created; result.pages_processed = r.pages; @@ -1011,6 +1025,11 @@ async function extractForSlugs( signal?: AbortSignal, // #1747/#1503: stamp resolved brain source id on batch rows (see ExtractOpts.sourceId). sourceId?: string, + // v0.42: when true, also extract frontmatter links per changed page so + // externally-edited YAML (`sources:`/`related:`) stays fresh on the cycle. + // Default false preserves the body-only incremental behavior. Gated upstream + // by `autopilot.incremental_extract_include_frontmatter`. + includeFrontmatter: boolean = false, ): Promise<{ links_created: number; timeline_created: number; pages: number }> { // Build the full slug set for link resolution (fast: just readdir, no file reads) const allFiles = walkMarkdownFiles(brainDir); @@ -1089,7 +1108,7 @@ async function extractForSlugs( const content = readFileSync(fullPath, 'utf-8'); if (doLinks) { - const links = await extractLinksFromFile(content, relPath, allSlugs, { globalBasename }); + const links = await extractLinksFromFile(content, relPath, allSlugs, { globalBasename, includeFrontmatter }); for (const link of links) { if (dryRun) { if (!jsonMode) console.log(` ${link.from_slug} → ${link.to_slug} (${link.link_type})`); diff --git a/src/core/config.ts b/src/core/config.ts index e2fdc971b..50fa2f723 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -146,6 +146,18 @@ export interface GBrainConfig { /** Daily spend cap (USD); bounds drains/day = floor(cap / ~$0.30). Default 2.0. */ max_usd_per_day?: number; }; + /** + * v0.42 — keep frontmatter links fresh on the incremental cycle. The cycle's + * extract phase re-extracts only the slugs a sync changed, but `extractForSlugs` + * extracts BODY links only — frontmatter (`sources:`/`related:` etc.) link edges + * silently drift stale when a page's YAML is edited externally and synced in. + * Set true to also extract frontmatter links per changed page each cycle, keeping + * externally-edited YAML edges fresh without a full rescan. Default false + * (preserves current behavior). Read via the file/env/DB plane in the cycle's + * extract dispatch. Disable/enable with + * `gbrain config set autopilot.incremental_extract_include_frontmatter `. + */ + incremental_extract_include_frontmatter?: boolean; }; eval?: { /** false disables capture entirely. Defaults to true. */ diff --git a/src/core/cycle.ts b/src/core/cycle.ts index a6a15fbc1..f98e370e8 100644 --- a/src/core/cycle.ts +++ b/src/core/cycle.ts @@ -1013,6 +1013,21 @@ async function runPhaseExtract( ): Promise { try { const { runExtractCore } = await import('../commands/extract.ts'); + const { loadConfig } = await import('./config.ts'); + // Default off: the incremental cycle extracts body links only unless the + // operator opts in to keeping externally-edited frontmatter links fresh too. + // Both planes, file wins (env > file > DB precedence, per loadConfigWithEngine): + // `gbrain config set autopilot.incremental_extract_include_frontmatter true` + // writes the DB plane (engine.setConfig), so a file-plane-only read here + // would make the documented enable command a silent no-op (#2120 class). + const fileVal = loadConfig()?.autopilot?.incremental_extract_include_frontmatter; + let includeFrontmatter = fileVal === true; + if (fileVal === undefined) { + try { + includeFrontmatter = + (await engine.getConfig('autopilot.incremental_extract_include_frontmatter')) === 'true'; + } catch { /* config table unreadable → default off */ } + } // Extract is read-mostly against the filesystem + write to links table. // Honor dryRun by skipping with a 'skipped' entry: extract doesn't have // a clean dry-run mode today and runCycle should be honest about it. @@ -1033,6 +1048,7 @@ async function runPhaseExtract( slugs: changedSlugs, // undefined = full walk (first run / manual) signal, sourceId, + includeFrontmatter, // honored on the incremental (slugs) path only }); const linksCreated = result?.links_created ?? 0; const timelineCreated = result?.timeline_entries_created ?? 0; diff --git a/src/core/link-extraction.ts b/src/core/link-extraction.ts index 4b8300d3c..41451abc3 100644 --- a/src/core/link-extraction.ts +++ b/src/core/link-extraction.ts @@ -960,8 +960,17 @@ export function makeResolver( const hints = Array.isArray(dirHint) ? dirHint : (dirHint ? [dirHint] : []); - // Step 1: already a slug? (dir/name shape, lowercase, hyphenated) - if (/^[a-z][a-z0-9-]*\/[a-z0-9][a-z0-9-]*$/.test(trimmed)) { + // Step 1: already a slug? Try an exact page lookup for any slug-shaped + // value (contains '/', slug charset). Broadened beyond the original + // single-segment lowercase-leading form (`^[a-z][a-z0-9-]*\/[a-z0-9]...`) + // to also accept digit-leading folders (`90-people/nicolai`, + // `01-trading/...`) and nested paths (`a/b/c`) — common in PARA-numbered + // vaults. This is an EXACT getPage match only — no fuzzy — so it never + // produces a false positive; a non-existent slug just falls through to + // the steps below. Fixes frontmatter `related: [[dir/slug]]` values + // (unwrapped by unwrapWikilink) that name a real page the strict regex + // could not reach and whose full-path fuzzy score is below threshold. + if (/\//.test(trimmed) && /^[a-z0-9][a-z0-9/_-]*$/.test(trimmed)) { const page = await engine.getPage(trimmed); if (page) { cache.set(cacheKey, trimmed); @@ -1025,6 +1034,25 @@ export function makeResolver( // ─── Frontmatter extractor ────────────────────────────────────── +/** + * Unwrap an Obsidian `[[wikilink]]` frontmatter value to its bare link + * target so the resolver (which expects bare titles / dir slugs) can match + * it. Mainstream Obsidian authors frontmatter links as `related: ["[[Page]]"]`; + * without this, the resolver treats the brackets as part of the value and a + * `[[90-people/nicolai]]` is normalized into `90peoplenicolai`, so it never + * resolves. Strips a trailing `|alias`, `#heading`, or `^block` suffix — the + * link target only. The regex is anchored to a wholly-wrapped value + * (`^\s*\[\[…\]\]\s*$`), so bare titles and any value not fully wrapped pass + * through unchanged and existing behavior is preserved exactly. + */ +export function unwrapWikilink(value: string): string { + const match = /^\s*\[\[(.+?)\]\]\s*$/.exec(value); + if (!match) return value; + // Take the link target: drop |alias, then #heading / ^block suffixes. + const target = match[1].split('|')[0].split('#')[0].split('^')[0]; + return target.trim(); +} + export interface UnresolvedFrontmatterRef { /** The frontmatter field name. */ field: string; @@ -1082,7 +1110,12 @@ export async function extractFrontmatterLinks( } if (!name) continue; // skip numbers, nulls, malformed objects - const resolved = await resolver.resolve(name, mapping.dirHint); + // Accept Obsidian `[[wikilink]]` values in frontmatter link fields by + // unwrapping to the bare target before resolution. Bare titles pass + // through unchanged; the original `name` is preserved for the + // unresolved report and edge context. + const linkTarget = unwrapWikilink(name); + const resolved = await resolver.resolve(linkTarget, mapping.dirHint); if (!resolved) { unresolved.push({ field, name }); continue; diff --git a/test/extract-incremental.test.ts b/test/extract-incremental.test.ts index dc47069f6..21307bb8a 100644 --- a/test/extract-incremental.test.ts +++ b/test/extract-incremental.test.ts @@ -236,3 +236,37 @@ describe('runExtractCore — incremental cycle path (#417)', () => { expect(result.links_created).toBeGreaterThan(0); }); }); +describe('runExtractCore — incremental frontmatter gate (includeFrontmatter)', () => { + // alice has a `source:` frontmatter edge but NO body links. The incremental + // path extracts body links only by default, so the frontmatter edge is the + // sole signal that distinguishes the gate off vs on. + const aliceFm = '---\nsource: companies/acme-example\n---\n# alice'; + + test('9. default (flag omitted) does NOT extract frontmatter links on the incremental path', async () => { + await seedPage('companies/acme-example', '# acme'); + await seedPage('people/alice-example', aliceFm); + const result = await runExtractCore(engine as unknown as BrainEngine, { + mode: 'all', + dir: tempDir, + slugs: ['people/alice-example'], + }); + // alice's only potential edge is her frontmatter `source:`; with the gate off + // it must not be extracted (preserves the body-only incremental behavior). + expect(result.pages_processed).toBe(1); + expect(result.links_created).toBe(0); + }); + + test('10. includeFrontmatter: true extracts the frontmatter link on the incremental path', async () => { + await seedPage('companies/acme-example', '# acme'); + await seedPage('people/alice-example', aliceFm); + const result = await runExtractCore(engine as unknown as BrainEngine, { + mode: 'all', + dir: tempDir, + slugs: ['people/alice-example'], + includeFrontmatter: true, + }); + // Same page, gate on → the `source:` frontmatter edge is now extracted. + expect(result.pages_processed).toBe(1); + expect(result.links_created).toBeGreaterThan(0); + }); +}); diff --git a/test/extract.test.ts b/test/extract.test.ts index 5764de52b..90cf54d1f 100644 --- a/test/extract.test.ts +++ b/test/extract.test.ts @@ -76,6 +76,18 @@ describe('extractLinksFromFile', () => { } }); + it('resolves wrapped [[wikilink]] digit-leading slug-path in frontmatter (fs resolver, broadened step 1)', async () => { + // Same bug class as makeResolver step 1 (#1983): the fs resolver's strict + // `^[a-z]…` slug regex rejected digit-leading / nested paths, so a PARA-vault + // `related: "[[90-people/nicolai]]"` never resolved even though the page exists. + const content = '---\nrelated: "[[90-people/nicolai]]"\ntype: concept\n---\nContent.'; + const allSlugs = new Set(['wiki/note', '90-people/nicolai']); + const links = await extractLinksFromFile(content, 'wiki/note.md', allSlugs, { includeFrontmatter: true }); + const related = links.filter(l => l.link_type === 'related_to'); + expect(related).toHaveLength(1); + expect(related[0].to_slug).toBe('90-people/nicolai'); + }); + it('frontmatter extraction is default OFF (back-compat)', async () => { // Without includeFrontmatter, fs-source no longer auto-extracts frontmatter. // Matches db-source behavior. User opts in with --include-frontmatter flag. diff --git a/test/link-extraction.test.ts b/test/link-extraction.test.ts index 9a2bc4f7d..bab980431 100644 --- a/test/link-extraction.test.ts +++ b/test/link-extraction.test.ts @@ -9,6 +9,7 @@ import { parseTimelineEntries, isAutoLinkEnabled, FRONTMATTER_LINK_MAP, + unwrapWikilink, type SlugResolver, } from '../src/core/link-extraction.ts'; import type { BrainEngine } from '../src/core/engine.ts'; @@ -1399,3 +1400,175 @@ describe('parseTimelineEntries — Format 3: inline [Source: ..., YYYY-MM-DD] ci expect(parseTimelineEntries('[Source: import batch, 2025-07-01]')).toHaveLength(0); }); }); +// ─── Frontmatter [[wikilink]] + slug-path resolution ────────────────────── +// Mainstream Obsidian authors frontmatter links as `related: ["[[Page]]"]`, +// and PARA-numbered vaults use digit-leading / nested slug paths like +// `[[90-people/nicolai]]`. Both were silently dropped: brackets were treated +// as part of the value and the step-1 slug regex (`^[a-z]…`) rejected +// digit-leading / nested paths, while full-path fuzzy scored below threshold. +// Fix: unwrapWikilink() before resolution + an exact getPage() for any +// slug-shaped value (exact-match only → no false positives). + +describe('unwrapWikilink', () => { + test('wrapped title → bare title', () => { + expect(unwrapWikilink('[[Monday Range]]')).toBe('Monday Range'); + }); + test('wrapped slug-path (digit-leading folder) → bare slug', () => { + expect(unwrapWikilink('[[90-people/nicolai]]')).toBe('90-people/nicolai'); + }); + test('wrapped nested slug-path → bare slug', () => { + expect(unwrapWikilink('[[01-trading/wiki/strategies/opening-range-breakout]]')) + .toBe('01-trading/wiki/strategies/opening-range-breakout'); + }); + test('strips |alias', () => { + expect(unwrapWikilink('[[90-people/nicolai|Nicolai]]')).toBe('90-people/nicolai'); + }); + test('strips #heading', () => { + expect(unwrapWikilink('[[Page#Section]]')).toBe('Page'); + }); + test('strips ^block', () => { + expect(unwrapWikilink('[[Page^abc123]]')).toBe('Page'); + }); + test('surrounding whitespace tolerated', () => { + expect(unwrapWikilink(' [[Page]] ')).toBe('Page'); + }); + test('bare title passes through unchanged', () => { + expect(unwrapWikilink('Monday Range')).toBe('Monday Range'); + }); + test('bare slug passes through unchanged', () => { + expect(unwrapWikilink('90-people/nicolai')).toBe('90-people/nicolai'); + }); + test('partially-wrapped value is NOT unwrapped (anchored)', () => { + // Not a wholly-wrapped value → left intact so existing behavior is exact. + expect(unwrapWikilink('see [[Page]] for detail')).toBe('see [[Page]] for detail'); + }); +}); + +describe('makeResolver — slug-path exact getPage (step 1 broadened)', () => { + function fakeEngine( + slugs: string[], + fuzzyMap: Map = new Map(), + ): BrainEngine { + const lookup = new Set(slugs); + return { + async getPage(slug: string) { return lookup.has(slug) ? { slug } as any : null; }, + async findByTitleFuzzy(name: string) { return fuzzyMap.get(name) ?? null; }, + async searchKeyword() { return []; }, + } as unknown as BrainEngine; + } + + test('digit-leading folder slug resolves via exact getPage', async () => { + const r = makeResolver(fakeEngine(['90-people/nicolai'])); + expect(await r.resolve('90-people/nicolai')).toBe('90-people/nicolai'); + }); + + test('nested (>2 segment) slug resolves via exact getPage', async () => { + const r = makeResolver(fakeEngine(['01-trading/wiki/strategies/opening-range-breakout'])); + expect(await r.resolve('01-trading/wiki/strategies/opening-range-breakout')) + .toBe('01-trading/wiki/strategies/opening-range-breakout'); + }); + + test('regression: single-segment lowercase slug still resolves', async () => { + const r = makeResolver(fakeEngine(['people/pedro'])); + expect(await r.resolve('people/pedro')).toBe('people/pedro'); + }); + + test('exact-only: slug-shaped value with no matching page falls through (no false positive)', async () => { + // `90-people/ghost` is slug-shaped but absent → step-1 getPage misses, + // no fuzzy hit → null. Never invents an edge. + const r = makeResolver(fakeEngine(['90-people/nicolai'])); + expect(await r.resolve('90-people/ghost')).toBeNull(); + }); + + test('non-slug value still routes to fuzzy', async () => { + const r = makeResolver(fakeEngine( + ['01-trading/monday-range'], + new Map([['Monday Range', { slug: '01-trading/monday-range', similarity: 1 }]]), + )); + expect(await r.resolve('Monday Range')).toBe('01-trading/monday-range'); + }); +}); + +describe('extractFrontmatterLinks — [[wikilink]] related: values (end-to-end)', () => { + function fakeEngine( + slugs: string[], + fuzzyMap: Map = new Map(), + ): BrainEngine { + const lookup = new Set(slugs); + return { + async getPage(slug: string) { return lookup.has(slug) ? { slug } as any : null; }, + async findByTitleFuzzy(name: string) { return fuzzyMap.get(name) ?? null; }, + async searchKeyword() { return []; }, + } as unknown as BrainEngine; + } + + test('wrapped slug-path related: resolves (the core win)', async () => { + const resolver = makeResolver(fakeEngine(['90-people/nicolai'])); + const { candidates, unresolved } = await extractFrontmatterLinks( + 'wiki/originals/ideas/note', 'note' as never, + { related: '[[90-people/nicolai]]' }, resolver, + ); + expect(unresolved).toHaveLength(0); + expect(candidates).toHaveLength(1); + expect(candidates[0]).toMatchObject({ + fromSlug: 'wiki/originals/ideas/note', + targetSlug: '90-people/nicolai', + linkType: 'related_to', + linkSource: 'frontmatter', + }); + }); + + test('wrapped nested slug-path related: resolves', async () => { + const resolver = makeResolver(fakeEngine(['01-trading/wiki/strategies/opening-range-breakout'])); + const { candidates } = await extractFrontmatterLinks( + 'wiki/note', 'note' as never, + { related: ['[[01-trading/wiki/strategies/opening-range-breakout]]'] }, resolver, + ); + expect(candidates).toHaveLength(1); + expect(candidates[0].targetSlug).toBe('01-trading/wiki/strategies/opening-range-breakout'); + }); + + test('wrapped value with |alias resolves to the target', async () => { + const resolver = makeResolver(fakeEngine(['90-people/nicolai'])); + const { candidates } = await extractFrontmatterLinks( + 'wiki/note', 'note' as never, + { related: '[[90-people/nicolai|Nicolai]]' }, resolver, + ); + expect(candidates).toHaveLength(1); + expect(candidates[0].targetSlug).toBe('90-people/nicolai'); + }); + + test('regression: bare slug related: still resolves', async () => { + const resolver = makeResolver(fakeEngine(['90-people/nicolai'])); + const { candidates } = await extractFrontmatterLinks( + 'wiki/note', 'note' as never, + { related: '90-people/nicolai' }, resolver, + ); + expect(candidates).toHaveLength(1); + expect(candidates[0].targetSlug).toBe('90-people/nicolai'); + }); + + test('regression: wrapped title resolves via fuzzy (brackets harmless)', async () => { + const resolver = makeResolver(fakeEngine( + ['01-trading/monday-range'], + new Map([['Monday Range', { slug: '01-trading/monday-range', similarity: 1 }]]), + )); + const { candidates } = await extractFrontmatterLinks( + 'wiki/note', 'note' as never, + { related: '[[Monday Range]]' }, resolver, + ); + expect(candidates).toHaveLength(1); + expect(candidates[0].targetSlug).toBe('01-trading/monday-range'); + }); + + test('unknown wrapped slug → unresolved (no crash), original value preserved', async () => { + const resolver = makeResolver(fakeEngine(['90-people/nicolai'])); + const { candidates, unresolved } = await extractFrontmatterLinks( + 'wiki/note', 'note' as never, + { related: '[[99-archive/does-not-exist]]' }, resolver, + ); + expect(candidates).toHaveLength(0); + expect(unresolved).toHaveLength(1); + expect(unresolved[0]).toEqual({ field: 'related', name: '[[99-archive/does-not-exist]]' }); + }); +}); From 800108e014028de6cfddaeef6af539582c1a0b09 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 16:45:29 -0700 Subject: [PATCH 04/39] =?UTF-8?q?v0.42.65.0=20chore(release):=2093=20verif?= =?UTF-8?q?ied=20fixes=20since=20v0.42.64.0=20=E2=80=94=20changelog=20+=20?= =?UTF-8?q?version=20bump=20(#3346)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(ci): refresh GitHub Actions SHA pins (checkout v4, action-gh-release v2) Pre-ship pin staleness check per docs/RELEASING.md: both floating major tags moved upstream; pins updated to the current tag commits. * v0.42.65.0 chore(release): 92 verified fixes since v0.42.64.0 — changelog + version bump Aggregates everything merged to master since the v0.42.64.0 bump commit: community fixes, credited takeovers, batch re-lands, CI hardening, and maintainer-approved features. Net commit list excludes revert pairs. No new schema migrations. Co-Authored-By: Claude Fable 5 * fix(deps): clear OSV-flagged transitive dependencies via override floors Raise the existing security-floor overrides so the lockfile resolves patched versions of three transitive packages flagged by the OSV scan (@hono/node-server, fast-uri, body-parser). None are on gbrain's own runtime path (@hono/node-server is only referenced by the MCP SDK's optional hono transport, which gbrain does not load); the floors keep the dependency scan green. MCP/OAuth unit tests pass against the resolved versions. * chore(release): fold #3110 into the v0.42.65.0 entry (93 net changes) --------- Co-authored-by: Garry Tan Co-authored-by: Claude Fable 5 --- .github/workflows/actionlint.yml | 2 +- .github/workflows/e2e.yml | 6 +- .github/workflows/heavy-tests.yml | 2 +- .github/workflows/release.yml | 4 +- .github/workflows/semgrep.yml | 2 +- .github/workflows/test.yml | 14 +-- CHANGELOG.md | 138 ++++++++++++++++++++++++++++++ VERSION | 2 +- bun.lock | 15 ++-- package.json | 7 +- 10 files changed, 168 insertions(+), 24 deletions(-) diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml index b4b6b8272..a6973ec4b 100644 --- a/.github/workflows/actionlint.yml +++ b/.github/workflows/actionlint.yml @@ -28,5 +28,5 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - uses: rhysd/actionlint@393031adb9afb225ee52ae2ccd7a5af5525e03e8 # v1.7.11 diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 5142a47c2..b59ecddb8 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -45,7 +45,7 @@ jobs: --health-timeout 5s --health-retries 5 steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: bun-version: 1.3.13 @@ -82,7 +82,7 @@ jobs: --health-timeout 5s --health-retries 5 steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: bun-version: 1.3.13 @@ -116,7 +116,7 @@ jobs: --health-timeout 5s --health-retries 5 steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: bun-version: 1.3.13 diff --git a/.github/workflows/heavy-tests.yml b/.github/workflows/heavy-tests.yml index bf21a88da..8d3b761f2 100644 --- a/.github/workflows/heavy-tests.yml +++ b/.github/workflows/heavy-tests.yml @@ -55,7 +55,7 @@ jobs: --health-timeout 5s --health-retries 5 steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: bun-version: 1.3.13 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e07a71c7d..9353f05b7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,7 +24,7 @@ jobs: id-token: write # for attest-build-provenance (Sigstore OIDC) attestations: write # for attest-build-provenance steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: bun-version: 1.3.13 @@ -49,7 +49,7 @@ jobs: with: path: artifacts - name: Create release - uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2 + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 with: files: | artifacts/gbrain-darwin-arm64/gbrain-darwin-arm64 diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index 7a736f4b0..d51ac05f1 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -26,7 +26,7 @@ jobs: container: image: semgrep/semgrep:1.170.0@sha256:c98f8829eea377274ee4b10656458b078b88232469b2ff913f091c2317347c9d steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 # Non-blocking initially (continue-on-error): the first runs establish a # baseline without failing unrelated PRs. Graduation path: once the # baseline findings are triaged (fixed or `# nosemgrep`'d), remove diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index db63724f8..3cd1647f8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,7 +43,7 @@ jobs: hit: ${{ steps.lookup.outputs.cache-hit }} hash: ${{ steps.compute.outputs.hash }} steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - name: Compute content hash id: compute run: | @@ -84,7 +84,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: fetch-depth: 0 - uses: gitleaks/gitleaks-action@dcedce43c6f43de0b836d1fe38946645c9c638dc # v2 @@ -103,7 +103,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 12 steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: bun-version: 1.3.13 @@ -124,7 +124,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: bun-version: 1.3.13 @@ -149,7 +149,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 12 steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: bun-version: 1.3.13 @@ -172,7 +172,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 12 steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: bun-version: 1.3.13 @@ -216,7 +216,7 @@ jobs: matrix: shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: bun-version: 1.3.13 diff --git a/CHANGELOG.md b/CHANGELOG.md index cb45f903c..1439ee6b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,144 @@ All notable changes to GBrain will be documented in this file. +## [0.42.65.0] - 2026-07-23 + +**A large maintenance release: 93 verified fixes and small features merged since v0.42.64.0, most of them community contributions.** + +If you use gbrain day to day, this release makes the boring parts trustworthy. Importing and syncing notes is safer: a failed pull no longer pretends everything is up to date, imported pages are read back after writing to confirm they landed, and a page with real content can no longer be silently overwritten by an empty one. Search answers get better inputs: the think command now picks excerpts that actually match your question, and results respect your federated source settings. Background enrichment (the "dream" cycle) wastes less money and retries properly when an AI provider is down. Spending caps now fail closed, so a billing hiccup can never turn into an uncapped spend. And `gbrain doctor` is quieter, with several false alarms removed and real problems (like an embedding backlog with no worker running) now flagged. + +More AI providers work out of the box, including OpenRouter prompt caching, MiniMax and Zhipu GLM recipes, Ollama Matryoshka embedding dimensions, and llama-server batch limits. + +## To take advantage of v0.42.65.0 + +`gbrain upgrade` should do this automatically. No new schema migrations ship in this release. + +1. **Upgrade and verify:** + ```bash + gbrain upgrade + gbrain doctor + gbrain stats + ``` +2. **If `gbrain doctor` reports new findings after upgrading,** that is the quieter, more accurate check set working as intended. Each finding names its fix. +3. **If any step fails,** please file an issue at https://github.com/garrytan/gbrain/issues with the output of `gbrain doctor` and `~/.gbrain/upgrade-errors.jsonl` if it exists. + +### Itemized changes + +#### Security + +- MCP source scoping for remote callers got a hardening pass, so agent-facing connections stay confined to the sources they were granted. (#2881, contributed by @spinsirr) +- Paid MCP spend accounting is now atomic and fails closed, and resolver spend is recorded before a cap error is raised, so caps cannot be raced past or undercounted. (#3203, #3204, contributed by @caterpillarC15) +- The OAuth token endpoint rate limit on the HTTP server is now configurable via env for deployments behind shared IPs. (#3114, contributed by @time-attack) +- `WWW-Authenticate` responses now carry `resource_metadata` per the MCP spec and RFC 9728, so conforming clients can discover the auth server. (#1410, contributed by @rayers) + +#### Search, retrieval, and think + +- `think` selects query-relevant excerpts instead of generic ones. (#3197, contributed by @Y0lan) +- Unqualified local CLI `search`/`query` now honors `sources.config.federated` read visibility. (#2561, #3141, contributed by @time-attack) +- Email citation metadata is projected into search results. (#2873, contributed by @amtagrwl) +- The `think` Gaps section renders once instead of twice. (#1662, contributed by @howwohmm) +- Fuzzy entity lookup threads the caller's source scope and skips soft-deleted entities. (#1508, contributed by @tim404x) +- `code-def` surfaces method, constructor, field, and struct definitions, not just top-level symbols. (#1628, contributed by @rayers) +- Briefing pages are excluded from their own Brain Pulse salience. (#1202, contributed by @rwbaker) +- Reranker calls with missing auth are classified as configuration errors before falling back. (#2059, #3139, contributed by @time-attack) + +#### Import, sync, and ingestion + +- A failed git pull with zero imports reports `partial (pull_failed)` instead of `up_to_date`. (#3068, #3253, contributed by @Masashi-Ono0611) +- Imports run a post-write read-back verification with a durable ingest-log record. (#2869, contributed by @Andredsouza1984) +- `put` refuses to overwrite a non-empty page with empty content. (#2708, contributed by @symmetric-matthew) +- `putPage` restores soft-deleted rows instead of colliding with them. (#2779, contributed by @RerankerGuo) +- Mixed-case slugs are normalized before chunk upsert, ending duplicate-chunk churn. (#430, #3143, contributed by @time-attack) +- Imports fall back to the body H1 for the title when frontmatter lacks `title:`. (#2446, #3072, contributed by @time-attack) +- YAML comments inside the frontmatter fence are no longer treated as markdown headings. (#3225, #3247, contributed by @Masashi-Ono0611) +- Write-through guards case-insensitive filesystem collisions before the atomic write. (#2831, #3119, contributed by @time-attack) +- Path-qualified wikilinks outside the known directory pattern resolve on the DB/put_page path. (#2866, contributed by @paul-0320) +- CJK slugs are supported in the slug registry and dream-cycle summary slugs. (#782, #738, #3083, contributed by @time-attack) +- Three ingest/sync/serve singleton fixes: page-type round-trip, deleted-slug embed noise, and a stateless width guard. (#3140, contributed by @time-attack) +- Sync honors the `embedding_disabled` sentinel as an implicit `--no-embed`. (#2879, contributed by @gawievanblerk) +- Verified sync head sentinels are cleared correctly. (#2734, contributed by @symmetric-matthew) +- Resumed syncs report the pinned commit they actually landed on. (#3202, contributed by @caterpillarC15) +- The expected `discover_git_root` probe failure stays off stderr. (#3232, contributed by @Masashi-Ono0611) +- `extract --stale` runs the real resolver so basename resolution reaches stale pages, and clears pre-version-bump pages. (#2576, #2717, contributed by @paul-0320; #1791, contributed by @Nazim22) +- Oversized code chunks are capped so they stay embeddable, and code-chunk metadata survives re-embeds. (#1675, contributed by @lubosxyz; #769, #1232, contributed by @rayers) + +#### Background cycle, dream, and facts + +- Path-derived dream sources are stamped, and the engine closes cleanly on autopilot shutdown. (#3178, contributed by @time-attack) +- All-provider-failed atom drains propagate so durable jobs retry instead of silently dropping work. (#3218, #3248, contributed by @Masashi-Ono0611) +- Atom extraction raises `maxTokens` and case-normalizes `atom_type` for Gemini models. (#3211, contributed by @alexey-metaengage) +- The conversation extractor gates anonymous-speaker self-attribution instead of guessing. (#3228, contributed by @asenkovskiy) +- Incremental dream extraction stamps its watermark so re-runs stop reprocessing. (#2636, #3115, contributed by @time-attack) +- `dream --dry-run --json` keeps stdout clean of embed summaries. (#394, #3109, contributed by @time-attack) +- Synthesized dream pages require a self-contained opening summary. (#2770, contributed by @Masashi-Ono0611) +- PGLite inline synth subagent drains complete, and `lint` gains `--exclude`. (#2699, #2649, #3162, contributed by @time-attack) +- Live context reads the documented "P1 Today" heading form with plain checkbox tasks, matching the daily-task-manager skill's output format. (#2186, #3124, contributed by @time-attack) +- Queued AI jobs refresh gateway config at execution time instead of using a stale snapshot. (#2125, contributed by @maxpetrusenkoagent) +- `brainstorm`/`propose_takes` honor configured models: cost preview uses the configured model, the judge reads its config key, provider probes are skipped when unneeded, and page projection is narrowed. (#3120, contributed by @time-attack) +- Backlog hardening wave: x-to-brain health check, propose_takes deadlines, capture title truncation, extract_atoms backlog handling, and pooler direct-URL routing. (#3165, contributed by @time-attack) +- `skillopt` emits `proposed.md` in no-mutate mode. (#2635, #3182, contributed by @time-attack) +- Nightly quality probe enable path and conversation-parser probe are wired up. (#2629, #2630, #3094, contributed by @time-attack) + +#### Doctor, health, and maintenance + +- New safe maintenance automation with a shared orphan-exclusion policy, so routine cleanup runs without risking linked content. (#3015, #3023, contributed by @time-attack) +- `orphan_ratio` excludes the chronicle volume under `life/events/`. (#2264, #3214, contributed by @asenkovskiy) +- `brain_score` orphan/timeline components use the orphans-audit linkable scope. (#3155, contributed by @time-attack) +- Entity timeline coverage is measured separately from whole-brain density. (#2761, contributed by @TurgutKural) +- Doctor flags embed backfills queued with no worker running. (#2696, contributed by @javieraldape) +- Two doctor false-positive/timeout fixes: the drift walk skips `node_modules`, and the bare-tweet check skips inline code and cited lines. (#1772, contributed by @sonlndv) +- A dead `llm_fallback_enabled` recommendation is dropped from conversation format coverage. (#1903, contributed by @ElliotDrel) +- Skill triggers with CRLF line endings parse on Windows. (#1149, contributed by @samporter-31) +- Onboard check names are registered in doctor categories, ending unknown-check warnings, and onboard-check remediations survive the `--apply --auto` path. (#3075, #3097, contributed by @time-attack) +- Dead slug prefixes are counted by slug. (#2697, contributed by @RerankerGuo) +- The backlinks worker defaults to check, not fix, and `check-backlinks` honors its positional directory argument. (#1853, contributed by @choomz; #3076, contributed by @time-attack) +- Calibration resolves the owner holder via config, defaulting to `self`. (#3077, contributed by @time-attack) +- Memory throttling on Linux reads `/proc/meminfo` MemAvailable. (#556, contributed by @chengzehsu) + +#### AI providers and gateway + +- OpenRouter gets family-scoped prompt caching, and query expansion works on chat-capable openai-compat recipes. (#3152, contributed by @time-attack) +- MiniMax recipe: embedding wire-shape compat fetch plus a chat touchpoint. (#1977, #3089, contributed by @time-attack) +- The Zhipu recipe gains a chat touchpoint so GLM subagents work. (#1157, #3084, contributed by @time-attack) +- Tier-configured models reach the recipe allowlist, Anthropic model lists are refreshed, tier resolutions are registered, and probe labels are honest. (#2800, contributed by @p3ob7o) +- Provider base URL config merges from the DB. (#1676, contributed by @TheLordArgus) +- The gateway falls back to the pooler when the derived direct host is unreachable. (#1641, #3088, contributed by @time-attack) +- Config-plane `voyage_api_key` folds into `VOYAGE_API_KEY` like the other hosted keys. (#3236, contributed by @Masashi-Ono0611) +- The `zeroentropyai:zerank-2` reranker has a pricing entry so the budget tracker can meter it. (#3223, #3233, contributed by @Masashi-Ono0611) +- llama-server embedding batches are capped at its 32-input request limit. (#1281, contributed by @mmekkaoui) +- Matryoshka dimensions thread through for Qwen3-Embedding on Ollama. (#1072, contributed by @mgandal) +- `init` seeds AI options from env on cold install, and `whoami` reports the stdio transport. (#3091, contributed by @time-attack) +- The `models` dispatch subcommand reads its first argument correctly. (#1428, contributed by @BenjaminDSmithy) +- Synopsis generation tail-truncates document text for small-model chat handlers. (#1427, contributed by @BenjaminDSmithy) +- The contradiction judge token cap is raised for thinking models. (#3210, contributed by @alexey-metaengage) + +#### Schema, migrations, and storage engines + +- Engine migration counts and surfaces per-page copy failures instead of silently advancing. (#3241, contributed by @Masashi-Ono0611) +- Invalid `CONCURRENTLY`-build index remnants are dropped without a DO block. (#3191, contributed by @Masashi-Ono0611) +- Unsupported large-dimension HNSW indexes are skipped instead of failing schema setup. (#1734, #3080, contributed by @time-attack) +- The v0.32.2 migration dirty-check scopes to targeted sources and surfaces failed phase detail. (#3093, contributed by @time-attack) +- Schema packs merge the full `extends` chain and `borrow_from` into the resolved manifest. (#1749, #3181, contributed by @time-attack) +- The schema-pack stats catch-all is narrowed so masked errors surface instead of fake zero-page counts. (#2466, #3133, contributed by @time-attack) +- Bundled schema-pack inspection reports the pack actually shipped in the binary, and minion subagent auth resolves through config. (#3110, contributed by @time-attack) +- PGLite `putPage` guards against zero-row RETURNING. (#1649, contributed by @alexhawkins) + +#### MCP server and CLI surface + +- `list_pages` rows include `source_id`. (#3209, contributed by @alexey-metaengage) +- Running CLI commands while `gbrain serve` (MCP) holds the brain now notifies about the conflict instead of failing confusingly. (#3243, contributed by @fdefitte) +- The OpenClaw plugin manifest entry is declared so the plugin loads. (#2551, #3185, contributed by @time-attack) + +#### For contributors + +- CI scanner roots are normalized on macOS. (#3198, contributed by @caterpillarC15) +- CI shard timeout raised to 22 minutes plus a delta-assert reporter leak test. (#3231, contributed by @time-attack) +- E2E suite hardening: flaky tests, no-op assertions, and cross-test coupling removed. (#1704, contributed by @auroracapital) +- `mechanical.test.ts` isolates `$HOME` so the E2E suite stops clobbering user config. (#434, contributed by @lloydarmbrust) +- The lint code-fence-wrap detector and fixer regex now agree. (#1597, contributed by @chungty) +- README project links for OpenClaw and Hermes are corrected. (#1961, #3179, contributed by @time-attack) +- A completed TODOS entry is dropped. (#3229, contributed by @Masashi-Ono0611) + ## [0.42.64.0] - 2026-07-20 ### Fixed diff --git a/VERSION b/VERSION index 610c068d6..bbceec6b1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.42.64.0 \ No newline at end of file +0.42.65.0 \ No newline at end of file diff --git a/bun.lock b/bun.lock index d7a03e8db..85cfdec80 100644 --- a/bun.lock +++ b/bun.lock @@ -51,8 +51,9 @@ "@electric-sql/pglite", ], "overrides": { - "@hono/node-server": "^1.19.13", - "fast-uri": "^3.1.2", + "@hono/node-server": "^2.0.5", + "body-parser": "^2.3.0", + "fast-uri": "^3.1.4", "fast-xml-builder": "^1.1.7", "fast-xml-parser": "^5.7.0", "form-data": "^4.0.6", @@ -162,7 +163,7 @@ "@electric-sql/pglite": ["@electric-sql/pglite@0.4.3", "", {}, "sha512-ichuWTgtd4mOM1G4SpyGJa5trT03lWbMypDV0fUXUCXg5hiHqVAz/bZyV68NqmkLB7WcYmj1RMJVSp8HV/v/ZQ=="], - "@hono/node-server": ["@hono/node-server@1.19.14", "", { "peerDependencies": { "hono": "^4" } }, "sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw=="], + "@hono/node-server": ["@hono/node-server@2.0.11", "", { "peerDependencies": { "hono": "^4" } }, "sha512-bjD221KPLoJTWUwso1J6fGKiTXEUFedG/s0visavY4zakFPkeGURMRNly+FhBHs7T8Dz4qHaZIMX9ZoJHSJtKA=="], "@jsquash/avif": ["@jsquash/avif@2.1.1", "", { "dependencies": { "wasm-feature-detect": "^1.2.11" } }, "sha512-LMRxd0fMgfCLtobDh0/sFYJMMiRJTNYSEEWvRDKXlAeZ08t3gI5V+1thIT0XjXJ+SVG7Zug9B0XPyx0Ti5VRNA=="], @@ -326,7 +327,7 @@ "asynckit": ["asynckit@0.4.0", "", {}, "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="], - "body-parser": ["body-parser@2.2.2", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.3", "http-errors": "^2.0.0", "iconv-lite": "^0.7.0", "on-finished": "^2.4.1", "qs": "^6.14.1", "raw-body": "^3.0.1", "type-is": "^2.0.1" } }, "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA=="], + "body-parser": ["body-parser@2.3.0", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^2.0.0", "debug": "^4.4.3", "http-errors": "^2.0.1", "iconv-lite": "^0.7.2", "on-finished": "^2.4.1", "qs": "^6.15.2", "raw-body": "^3.0.2", "type-is": "^2.1.0" } }, "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw=="], "bowser": ["bowser@2.14.1", "", {}, "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg=="], @@ -400,7 +401,7 @@ "fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="], - "fast-uri": ["fast-uri@3.1.3", "", {}, "sha512-i70LwGWUduXqzicKXWshooq+sWL1K3WUU5rKZNG/0i3a1OSoX3HqhH5WbWwTmqWfor4urUakGPiRQcleRZTwOg=="], + "fast-uri": ["fast-uri@3.1.4", "", {}, "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw=="], "fast-xml-builder": ["fast-xml-builder@1.3.0", "", { "dependencies": { "path-expression-matcher": "^1.6.2", "xml-naming": "^0.3.0" } }, "sha512-F74cZEdCvuw9P41GAC3rod4X04jjWGM1JPEv/GWSqFTWLsdyMSBMBMlm9Hk3GLBgLBbdBNY8yee0pQh2RBVESQ=="], @@ -614,6 +615,10 @@ "@types/bun/bun-types": ["bun-types@1.3.11", "", { "dependencies": { "@types/node": "*" } }, "sha512-1KGPpoxQWl9f6wcZh57LvrPIInQMn2TQ7jsgxqpRzg+l0QPOFvJVH7HmvHo/AiPgwXy+/Thf6Ov3EdVn1vOabg=="], + "body-parser/content-type": ["content-type@2.0.0", "", {}, "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ=="], + + "body-parser/type-is": ["type-is@2.1.0", "", { "dependencies": { "content-type": "^2.0.0", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA=="], + "es-set-tostringtag/hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="], "eventsource/eventsource-parser": ["eventsource-parser@3.0.6", "", {}, "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg=="], diff --git a/package.json b/package.json index 4d38e1255..80d1a5e78 100644 --- a/package.json +++ b/package.json @@ -144,10 +144,11 @@ "bun": ">=1.3.10" }, "license": "MIT", - "version": "0.42.64.0", + "version": "0.42.65.0", "overrides": { - "@hono/node-server": "^1.19.13", - "fast-uri": "^3.1.2", + "@hono/node-server": "^2.0.5", + "fast-uri": "^3.1.4", + "body-parser": "^2.3.0", "fast-xml-builder": "^1.1.7", "fast-xml-parser": "^5.7.0", "form-data": "^4.0.6", From 03cd52631b51506fa665239373c62882ced32aee Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 16:45:36 -0700 Subject: [PATCH 05/39] fix(thin-client): map --source scope onto source_id for remote-routed ops (#3086) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(thin-client): map --source/GBRAIN_SOURCE/.gbrain-source onto source_id for routed ops (#2098) The thin-client route short-circuits before makeContext, so the 6-tier source resolution never ran and `gbrain query --source X` against a remote brain sent the unknown `source` key verbatim — the server op ignored it and searched unscoped. applyThinClientSourceScope now runs the engine-free tiers (flag → env → dotfile; DB-backed tiers need an engine, and the server's grant scoping covers the rest) and sets the op's source_id wire param. Ops declaring their own `source` param are untouched; an explicit --source on an op with no source_id wire param errors loudly instead of silently dropping; explicit --source-id/--all-sources on the wire win over ambient tiers. Fixes #2098 Co-Authored-By: Claude Fable 5 * test(thin-client): use withEnv() instead of direct process.env mutation (test-isolation R1) CI verify failed on check:test-isolation — thin-client-source-scope.test.ts mutated process.env.GBRAIN_SOURCE via beforeEach/afterEach. Wrapped each test body in withEnv() from test/helpers/with-env.ts instead. Co-Authored-By: Claude Fable 5 * fix(thin-client): keep ambient scope out of get_skill's non-scope source_id param get_skill's source_id is a mode switch (host catalog vs brain-resident-pack lookup), not a read-scope filter. Ambient GBRAIN_SOURCE / .gbrain-source injection would silently reroute 'gbrain skill ' on thin clients to getResidentSkillDetail. Exclude it via NON_SCOPE_SOURCE_ID_OPS; explicit --source-id still passes through, explicit --source errors with a hint. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Sinabina Co-authored-by: Claude Fable 5 Co-authored-by: Garry Tan --- src/cli.ts | 64 ++++++++++++ src/core/source-resolver.ts | 27 +++++ test/thin-client-source-scope.test.ts | 142 ++++++++++++++++++++++++++ 3 files changed, 233 insertions(+) create mode 100644 test/thin-client-source-scope.test.ts diff --git a/src/cli.ts b/src/cli.ts index 49a96e6df..b12b776f0 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -24,6 +24,7 @@ import type { GBrainConfig } from './core/config.ts'; import type { AIGatewayConfig } from './core/ai/types.ts'; import type { BrainEngine } from './core/engine.ts'; import { operations, OperationError } from './core/operations.ts'; +import { resolveSourceIdEngineFree } from './core/source-resolver.ts'; import { formatVolunteeredPage } from './core/context/volunteer.ts'; import type { Operation, OperationContext } from './core/operations.ts'; import { shouldForceExitAfterMain, finishCliTeardown, flushThenExit, currentExitCode, setCliExitVerdict } from './core/cli-force-exit.ts'; @@ -384,6 +385,15 @@ async function main() { if (op.localOnly) { refuseThinClient(command, cfgPre!.remote_mcp!.mcp_url); } + // #2098: the local path resolves --source / GBRAIN_SOURCE / .gbrain-source + // inside makeContext (ctx.sourceId), which this route never reaches — so + // scope must be mapped onto the op's source_id wire param before the call. + try { + applyThinClientSourceScope(op, params); + } catch (e: unknown) { + console.error(e instanceof Error ? e.message : String(e)); + process.exit(1); + } await runThinClientRouted(op, params, cfgPre!, cliOpts); return; } @@ -804,6 +814,60 @@ export function parseOpArgs(op: Operation, args: string[]): Record, + cwd?: string, +): void { + if ('source' in op.params) return; // the op owns --source; not a scope flag + const explicit = typeof params.source === 'string' && params.source.length > 0 + ? (params.source as string) + : null; + delete params.source; // never a wire param on these ops — don't leak it + // Explicit per-call scope already on the wire wins over ambient tiers. + if (params.source_id !== undefined || params.all_sources === true) { + if (explicit) { + throw new Error('Pass either --source or --source-id/--all-sources, not both.'); + } + return; + } + const resolved = resolveSourceIdEngineFree(explicit, cwd); + if (!resolved) return; + if (!('source_id' in op.params) || NON_SCOPE_SOURCE_ID_OPS.has(op.name)) { + if (explicit) { + const hint = NON_SCOPE_SOURCE_ID_OPS.has(op.name) + ? `(its source_id parameter is not a scope filter; pass --source-id explicitly if you mean it)` + : `(the remote op has no source_id parameter; the server scopes it to your grant)`; + throw new Error( + `gbrain ${op.cliHints?.name || op.name} does not accept --source on a thin-client install ${hint}.`, + ); + } + return; // ambient env/dotfile scope with nowhere to send it + } + params.source_id = resolved; +} + async function makeContext(engine: BrainEngine, params: Record): Promise { // v0.31.8 (D11): resolve sourceId via the canonical 6-tier chain. Honors // --source / GBRAIN_SOURCE / .gbrain-source / path-match / brain default / diff --git a/src/core/source-resolver.ts b/src/core/source-resolver.ts index d81d3878f..03f9eb0c0 100644 --- a/src/core/source-resolver.ts +++ b/src/core/source-resolver.ts @@ -160,6 +160,33 @@ export async function resolveSourceId( return 'default'; } +/** + * Engine-free tiers (1-3) of the resolution chain: explicit flag → + * GBRAIN_SOURCE env → .gbrain-source dotfile walk. Used by the thin-client + * CLI path (#2098), which has no local engine to run tiers 4-6 or + * assertSourceExists against — the remote server enforces existence + grant. + * Returns null when no engine-free tier fires. + */ +export function resolveSourceIdEngineFree( + explicit: string | null | undefined, + cwd: string = process.cwd(), +): string | null { + if (explicit) { + if (!SOURCE_ID_RE.test(explicit)) { + throw new Error(`Invalid --source value "${explicit}". Must match [a-z0-9-]{1,32}.`); + } + return explicit; + } + const env = process.env.GBRAIN_SOURCE; + if (env && env.length > 0) { + if (!SOURCE_ID_RE.test(env)) { + throw new Error(`Invalid GBRAIN_SOURCE value "${env}". Must match [a-z0-9-]{1,32}.`); + } + return env; + } + return readDotfileWalk(cwd); +} + /** * Returns the id of the SINGLE registered non-default source with a * local_path, when exactly one such row exists. Returns null when: diff --git a/test/thin-client-source-scope.test.ts b/test/thin-client-source-scope.test.ts new file mode 100644 index 000000000..d366f319d --- /dev/null +++ b/test/thin-client-source-scope.test.ts @@ -0,0 +1,142 @@ +/** + * #2098: thin-client routing dropped --source / GBRAIN_SOURCE / .gbrain-source. + * + * The local CLI path resolves source scope in makeContext (ctx.sourceId); the + * thin-client route short-circuits before that and sent params verbatim, so + * `gbrain query --source X` against a remote brain silently searched unscoped + * (the server op ignores the unknown `source` key). + * + * applyThinClientSourceScope runs the engine-free tiers (flag → env → dotfile) + * and maps the result onto the op's `source_id` wire param. These tests fail + * without the fix (params.source_id stays undefined / params.source leaks). + */ + +import { describe, test, expect } from 'bun:test'; +import { mkdtempSync, rmSync, writeFileSync } from 'fs'; +import { join } from 'path'; +import { tmpdir } from 'os'; +import { applyThinClientSourceScope, parseOpArgs } from '../src/cli.ts'; +import { operationsByName } from '../src/core/operations.ts'; +import { withEnv } from './helpers/with-env.ts'; + +const queryOp = operationsByName.query; + +describe('applyThinClientSourceScope (#2098)', () => { + test('--source maps onto the query op wire param source_id', async () => { + await withEnv({ GBRAIN_SOURCE: undefined }, () => { + const params = parseOpArgs(queryOp, ['find things', '--source', 'wiki']); + expect(params.source).toBe('wiki'); // pre-fix state: wrong key + applyThinClientSourceScope(queryOp, params, '/'); + expect(params.source_id).toBe('wiki'); + expect('source' in params).toBe(false); // never leaks the unknown key + }); + }); + + test('GBRAIN_SOURCE env tier fires when no flag is passed', async () => { + await withEnv({ GBRAIN_SOURCE: 'gstack' }, () => { + const params = parseOpArgs(queryOp, ['find things']); + applyThinClientSourceScope(queryOp, params, '/'); + expect(params.source_id).toBe('gstack'); + }); + }); + + test('.gbrain-source dotfile tier fires when flag and env are absent', async () => { + await withEnv({ GBRAIN_SOURCE: undefined }, () => { + const tmp = mkdtempSync(join(tmpdir(), 'gbrain-thin-scope-')); + try { + writeFileSync(join(tmp, '.gbrain-source'), 'essays\n'); + const params = parseOpArgs(queryOp, ['find things']); + applyThinClientSourceScope(queryOp, params, tmp); + expect(params.source_id).toBe('essays'); + } finally { + rmSync(tmp, { recursive: true, force: true }); + } + }); + }); + + test('explicit --source-id on the wire wins over ambient env scope', async () => { + await withEnv({ GBRAIN_SOURCE: 'gstack' }, () => { + const params = parseOpArgs(queryOp, ['find things', '--source-id', 'wiki']); + applyThinClientSourceScope(queryOp, params, '/'); + expect(params.source_id).toBe('wiki'); + }); + }); + + test('--source together with --source-id is rejected loudly', async () => { + await withEnv({ GBRAIN_SOURCE: undefined }, () => { + const params = parseOpArgs(queryOp, ['q', '--source', 'a', '--source-id', 'b']); + expect(() => applyThinClientSourceScope(queryOp, params, '/')).toThrow(/not both/); + }); + }); + + test('invalid --source value is rejected loudly', async () => { + await withEnv({ GBRAIN_SOURCE: undefined }, () => { + const params = parseOpArgs(queryOp, ['q', '--source', 'Bad_Value!']); + expect(() => applyThinClientSourceScope(queryOp, params, '/')).toThrow(/Invalid --source/); + }); + }); + + test('--source on an op with no source_id wire param errors instead of silently dropping', async () => { + await withEnv({ GBRAIN_SOURCE: undefined }, () => { + const op = operationsByName.add_tag; + expect('source_id' in op.params).toBe(false); + const params = { slug: 'x', tag: 'y', source: 'wiki' }; + expect(() => applyThinClientSourceScope(op, params, '/')).toThrow(/--source/); + }); + }); + + test('ambient env scope on an op with no source_id wire param is ignored (no throw)', async () => { + await withEnv({ GBRAIN_SOURCE: 'wiki' }, () => { + const op = operationsByName.add_tag; + const params: Record = { slug: 'x', tag: 'y' }; + applyThinClientSourceScope(op, params, '/'); + expect(params.source_id).toBeUndefined(); + }); + }); + + test('ops that declare their OWN source param are left untouched', async () => { + await withEnv({ GBRAIN_SOURCE: undefined }, () => { + const op = operationsByName.put_raw_data; + expect('source' in op.params).toBe(true); + const params: Record = { slug: 'x', source: 'crustdata', data: {} }; + applyThinClientSourceScope(op, params, '/'); + expect(params.source).toBe('crustdata'); + expect(params.source_id).toBeUndefined(); + }); + }); + + test('get_skill: ambient scope never leaks into its non-scope source_id param', async () => { + await withEnv({ GBRAIN_SOURCE: 'wiki' }, () => { + const op = operationsByName.get_skill; + expect('source_id' in op.params).toBe(true); // has the param, but it is a mode switch + const params: Record = { name: 'ingest' }; + applyThinClientSourceScope(op, params, '/'); + expect(params.source_id).toBeUndefined(); // would flip host catalog → brain-pack lookup + }); + }); + + test('get_skill: explicit --source errors instead of masquerading as --source-id', async () => { + await withEnv({ GBRAIN_SOURCE: undefined }, () => { + const op = operationsByName.get_skill; + const params: Record = { name: 'ingest', source: 'wiki' }; + expect(() => applyThinClientSourceScope(op, params, '/')).toThrow(/--source-id/); + }); + }); + + test('get_skill: explicit --source-id passes through untouched', async () => { + await withEnv({ GBRAIN_SOURCE: 'gstack' }, () => { + const op = operationsByName.get_skill; + const params: Record = { name: 'ingest', source_id: 'wiki' }; + applyThinClientSourceScope(op, params, '/'); + expect(params.source_id).toBe('wiki'); + }); + }); + + test('no scope from any tier leaves params unchanged', async () => { + await withEnv({ GBRAIN_SOURCE: undefined }, () => { + const params = parseOpArgs(queryOp, ['find things']); + applyThinClientSourceScope(queryOp, params, '/'); + expect(params.source_id).toBeUndefined(); + }); + }); +}); From d89d6ea2938af5c048a74acfbfa8cc8a040fcc7b Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:05:12 -0700 Subject: [PATCH 06/39] fix(doctor): scope timeline labels to disambiguate entity coverage vs brain-score component (#2298) (#3073) doctor and the get_health CLI surface printed two different timeline metrics under one ambiguous 'timeline' label: the entity-scoped timeline_coverage fraction (eligible entity pages with a timeline entry) and the whole-brain timeline_coverage_score brain-score component (all pages with a timeline entry, 0-15). Different numerators AND denominators, indistinguishable in output. Label-only fix, scoring unchanged: - graph_coverage check: 'entity timeline coverage N%' - brain_score breakdown: 'timeline density (all pages) N/15' - get_health CLI: 'Timeline coverage (entity pages)' plus a new 'Timeline density (all pages): N/15' line when the score is present Adds test/doctor-timeline-metric-labels-2298.test.ts pinning the denominator semantics, the rendered doctor messages, and the CLI guard matrix (fails on master, passes here). Takeover of #2761. Co-authored-by: Sinabina Co-authored-by: TurgutKural Co-authored-by: Claude Fable 5 From ca4cf2a0c6a4e0a53f20c49cbe77a64ada01a73e Mon Sep 17 00:00:00 2001 From: RP-AGENT-BOT Date: Thu, 23 Jul 2026 19:12:57 -0500 Subject: [PATCH 07/39] fix(cycle): preserve per-page multi-claim proposals (#3297) Co-authored-by: David Guidry --- src/core/cycle/propose-takes.ts | 14 +++-- src/core/migrate.ts | 14 +++++ src/core/pglite-schema.ts | 2 +- src/core/schema-embedded.ts | 7 +-- src/schema.sql | 7 +-- test/propose-takes-per-claim.test.ts | 93 ++++++++++++++++++++++++++++ test/propose-takes.test.ts | 21 ++++++- 7 files changed, 142 insertions(+), 16 deletions(-) create mode 100644 test/propose-takes-per-claim.test.ts diff --git a/src/core/cycle/propose-takes.ts b/src/core/cycle/propose-takes.ts index 4fe1f1b2a..e6117fc02 100644 --- a/src/core/cycle/propose-takes.ts +++ b/src/core/cycle/propose-takes.ts @@ -484,16 +484,18 @@ class ProposeTakesPhase extends BaseCyclePhase { continue; } - // Write proposals to take_proposals. Each row is a separate INSERT - // because the composite idempotency key is on the per-page tuple — a - // bulk UPSERT would collapse a same-page-multi-claim run into one row. + // Write proposals to take_proposals. #2138: the idempotency key is + // per-CLAIM — take_proposals_idempotency_idx folds md5(claim_text) into + // the per-page tuple (migration v125), so a multi-claim page keeps every + // claim. RETURNING id prevents a repeated claim from inflating the count. for (const p of proposals) { - await engine.executeRaw( + const inserted = await engine.executeRaw<{ id: number }>( `INSERT INTO take_proposals (source_id, page_slug, content_hash, prompt_version, proposal_run_id, claim_text, kind, holder, weight, domain, dedup_against_fence_rows, model_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) - ON CONFLICT (source_id, page_slug, content_hash, prompt_version) DO NOTHING`, + ON CONFLICT (source_id, page_slug, content_hash, prompt_version, md5(claim_text)) DO NOTHING + RETURNING id`, [ sourceId, page.slug, @@ -509,7 +511,7 @@ class ProposeTakesPhase extends BaseCyclePhase { modelId, ], ); - result.proposals_inserted += 1; + result.proposals_inserted += inserted.length; } } diff --git a/src/core/migrate.ts b/src/core/migrate.ts index e0caafbe2..085368246 100644 --- a/src/core/migrate.ts +++ b/src/core/migrate.ts @@ -5710,6 +5710,20 @@ export const MIGRATIONS: Migration[] = [ `); }, }, + { + version: 125, + name: 'take_proposals_per_claim_idempotency', + // #2138: the original idempotency key was per page, so each INSERT after + // the first claim silently conflicted. md5(claim_text) makes it per claim + // without adding/backfilling a column. The new key is strictly finer than + // the old key and therefore preserves all existing rows. + idempotent: true, + sql: ` + DROP INDEX IF EXISTS take_proposals_idempotency_idx; + CREATE UNIQUE INDEX IF NOT EXISTS take_proposals_idempotency_idx + ON take_proposals (source_id, page_slug, content_hash, prompt_version, md5(claim_text)); + `, + }, ]; export const LATEST_VERSION = MIGRATIONS.length > 0 diff --git a/src/core/pglite-schema.ts b/src/core/pglite-schema.ts index a42837a7c..db1c762d5 100644 --- a/src/core/pglite-schema.ts +++ b/src/core/pglite-schema.ts @@ -777,7 +777,7 @@ CREATE TABLE IF NOT EXISTS take_proposals ( predicted_brier_bucket_n INTEGER ); CREATE UNIQUE INDEX IF NOT EXISTS take_proposals_idempotency_idx - ON take_proposals (source_id, page_slug, content_hash, prompt_version); + ON take_proposals (source_id, page_slug, content_hash, prompt_version, md5(claim_text)); CREATE INDEX IF NOT EXISTS take_proposals_pending_idx ON take_proposals (source_id, status, proposed_at DESC) WHERE status = 'pending'; diff --git a/src/core/schema-embedded.ts b/src/core/schema-embedded.ts index dd0362bd5..0e7e2c48e 100644 --- a/src/core/schema-embedded.ts +++ b/src/core/schema-embedded.ts @@ -1273,9 +1273,8 @@ CREATE INDEX IF NOT EXISTS calibration_profiles_published_idx ON calibration_profiles (source_id, published, holder) WHERE published = true; --- take_proposals: propose_takes phase queue. Idempotency cache via the --- composite unique index (source_id, page_slug, content_hash, prompt_version) --- mirrors v0.23 dream_verdicts. proposal_run_id supports --rollback by run. +-- take_proposals: per-claim idempotency via source/page/content/prompt plus +-- md5(claim_text). The old per-page key silently dropped claim #2+ (#2138). CREATE TABLE IF NOT EXISTS take_proposals ( id BIGSERIAL PRIMARY KEY, source_id TEXT NOT NULL REFERENCES sources(id) ON DELETE CASCADE, @@ -1301,7 +1300,7 @@ CREATE TABLE IF NOT EXISTS take_proposals ( predicted_brier_bucket_n INTEGER ); CREATE UNIQUE INDEX IF NOT EXISTS take_proposals_idempotency_idx - ON take_proposals (source_id, page_slug, content_hash, prompt_version); + ON take_proposals (source_id, page_slug, content_hash, prompt_version, md5(claim_text)); CREATE INDEX IF NOT EXISTS take_proposals_pending_idx ON take_proposals (source_id, status, proposed_at DESC) WHERE status = 'pending'; diff --git a/src/schema.sql b/src/schema.sql index 8c8faa224..084af8d7e 100644 --- a/src/schema.sql +++ b/src/schema.sql @@ -1269,9 +1269,8 @@ CREATE INDEX IF NOT EXISTS calibration_profiles_published_idx ON calibration_profiles (source_id, published, holder) WHERE published = true; --- take_proposals: propose_takes phase queue. Idempotency cache via the --- composite unique index (source_id, page_slug, content_hash, prompt_version) --- mirrors v0.23 dream_verdicts. proposal_run_id supports --rollback by run. +-- take_proposals: per-claim idempotency via source/page/content/prompt plus +-- md5(claim_text). The old per-page key silently dropped claim #2+ (#2138). CREATE TABLE IF NOT EXISTS take_proposals ( id BIGSERIAL PRIMARY KEY, source_id TEXT NOT NULL REFERENCES sources(id) ON DELETE CASCADE, @@ -1297,7 +1296,7 @@ CREATE TABLE IF NOT EXISTS take_proposals ( predicted_brier_bucket_n INTEGER ); CREATE UNIQUE INDEX IF NOT EXISTS take_proposals_idempotency_idx - ON take_proposals (source_id, page_slug, content_hash, prompt_version); + ON take_proposals (source_id, page_slug, content_hash, prompt_version, md5(claim_text)); CREATE INDEX IF NOT EXISTS take_proposals_pending_idx ON take_proposals (source_id, status, proposed_at DESC) WHERE status = 'pending'; diff --git a/test/propose-takes-per-claim.test.ts b/test/propose-takes-per-claim.test.ts new file mode 100644 index 000000000..79e551fb9 --- /dev/null +++ b/test/propose-takes-per-claim.test.ts @@ -0,0 +1,93 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test'; +import { PGLiteEngine } from '../src/core/pglite-engine.ts'; +import { resetPgliteState } from './helpers/reset-pglite.ts'; +import { + runPhaseProposeTakes, + type ProposeTakesExtractor, +} from '../src/core/cycle/propose-takes.ts'; +import { MIGRATIONS } from '../src/core/migrate.ts'; +import type { OperationContext } from '../src/core/operations.ts'; + +let engine: PGLiteEngine; + +beforeAll(async () => { + engine = new PGLiteEngine(); + await engine.connect({}); + await engine.initSchema(); +}); + +afterAll(async () => { + await engine.disconnect(); +}); + +beforeEach(async () => { + await resetPgliteState(engine); +}); + +function context(): OperationContext { + return { + engine, + config: {} as never, + logger: { info() {}, warn() {}, error() {} } as never, + dryRun: false, + remote: false, + sourceId: 'default', + }; +} + +async function countProposals(slug: string): Promise { + const rows = await engine.executeRaw<{ n: string }>( + `SELECT COUNT(*)::text AS n + FROM take_proposals + WHERE page_slug = $1 AND source_id = 'default'`, + [slug], + ); + return Number(rows[0]!.n); +} + +const proposals: ProposeTakesExtractor = async () => [ + { claim_text: 'Claim one', kind: 'take', holder: 'brain', weight: 0.6 }, + { claim_text: 'Claim two', kind: 'bet', holder: 'brain', weight: 0.8 }, + { claim_text: 'Claim one', kind: 'take', holder: 'brain', weight: 0.6 }, +]; + +async function putThesis(): Promise { + await engine.putPage('wiki/essays/thesis', { + title: 'thesis', + type: 'analysis' as never, + compiled_truth: 'Two strong claims live in this essay.', + frontmatter: {}, + timeline: '', + }); +} + +describe('#2138 per-claim proposal idempotency', () => { + test('keeps distinct claims, drops repeated claim, then page-cache hits', async () => { + await putThesis(); + const result = await runPhaseProposeTakes(context(), { extractor: proposals }); + expect((result.details as Record).proposals_inserted).toBe(2); + expect(await countProposals('wiki/essays/thesis')).toBe(2); + + const rerun = await runPhaseProposeTakes(context(), { extractor: proposals }); + expect((rerun.details as Record).cache_hits).toBe(1); + expect(await countProposals('wiki/essays/thesis')).toBe(2); + }); + + test('migration v125 replaces the old-shaped index', async () => { + await engine.executeRaw('DROP INDEX IF EXISTS take_proposals_idempotency_idx'); + await engine.executeRaw( + `CREATE INDEX take_proposals_idempotency_idx + ON take_proposals (source_id, page_slug, content_hash, prompt_version)`, + ); + const migration = MIGRATIONS.find((entry) => entry.version === 125); + expect(migration).toBeDefined(); + for (const statement of migration!.sql!.split(';').map(value => value.trim()).filter(Boolean)) { + await engine.executeRaw(statement); + } + + await putThesis(); + const result = await runPhaseProposeTakes(context(), { extractor: proposals }); + expect((result.details as Record).proposals_inserted).toBe(2); + expect(await countProposals('wiki/essays/thesis')).toBe(2); + }); +}); diff --git a/test/propose-takes.test.ts b/test/propose-takes.test.ts index fbe8deb26..17a2e1d47 100644 --- a/test/propose-takes.test.ts +++ b/test/propose-takes.test.ts @@ -68,7 +68,10 @@ function buildMockEngine(opts: { if (existing.has(key)) return [{ id: 1 } as unknown as T]; return []; } - // INSERT — return nothing + // INSERT ... RETURNING id — one row per successful insert (#2138). + if (sql.includes('INSERT INTO take_proposals')) { + return [{ id: captured.length } as unknown as T]; + } return []; }, } as unknown as BrainEngine; @@ -276,6 +279,22 @@ describe('runPhaseProposeTakes — phase integration', () => { expect(inserts[0]!.params[9]).toBe('market'); // domain }); + test('#2138: multi-claim page inserts every claim with a per-claim conflict target', async () => { + const pages = [buildPage({ slug: 'wiki/essays/thesis', body: 'Two strong claims live here.' })]; + const { engine, captured } = buildMockEngine({ pages }); + const extractor: ProposeTakesExtractor = async () => [ + { claim_text: 'Claim one', kind: 'take', holder: 'brain', weight: 0.6 }, + { claim_text: 'Claim two', kind: 'bet', holder: 'brain', weight: 0.8 }, + ]; + const result = await runPhaseProposeTakes(buildCtx(engine), { extractor }); + + expect((result.details as Record).proposals_inserted).toBe(2); + const inserts = captured.filter(c => c.sql.includes('INSERT INTO take_proposals')); + expect(inserts).toHaveLength(2); + for (const insert of inserts) expect(insert.sql).toContain('md5(claim_text)'); + expect(inserts.map(i => i.params[5])).toEqual(['Claim one', 'Claim two']); + }); + test('cache hit: page already in take_proposals is skipped', async () => { const body = 'A page that was already processed.'; const pages = [buildPage({ slug: 'wiki/old-page', body })]; From be722ee5f716fbfc7e0d5605d7d14647188deb1e Mon Sep 17 00:00:00 2001 From: arisgysel-design Date: Fri, 24 Jul 2026 02:13:03 +0200 Subject: [PATCH 08/39] fix(takes): query active row before superseding (#3275) Fixes #2663. Co-authored-by: arisgysel-design --- src/commands/takes.ts | 2 +- test/takes-command-source-scope.test.ts | 46 ++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/src/commands/takes.ts b/src/commands/takes.ts index 4ef7e82f0..981463822 100644 --- a/src/commands/takes.ts +++ b/src/commands/takes.ts @@ -292,7 +292,7 @@ async function cmdSupersede(engine: BrainEngine, args: string[], sourceId?: stri const pageId = await getPageId(engine, slug, sourceId); // Read existing row to inherit kind/holder unless overridden - const existing = await engine.listTakes({ page_id: pageId, active: false, limit: 500 }); + const existing = await engine.listTakes({ page_id: pageId, active: true, limit: 500 }); const target = existing.find(t => t.row_num === rowNum); if (!target) { console.error(`Row #${rowNum} not found on ${slug}.`); diff --git a/test/takes-command-source-scope.test.ts b/test/takes-command-source-scope.test.ts index 3c970be08..85c714bcf 100644 --- a/test/takes-command-source-scope.test.ts +++ b/test/takes-command-source-scope.test.ts @@ -3,7 +3,7 @@ import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { runTakes } from '../src/commands/takes.ts'; -import type { BrainEngine, TakeBatchInput } from '../src/core/engine.ts'; +import type { BrainEngine, TakeBatchInput, TakesListOpts } from '../src/core/engine.ts'; import { withEnv } from './helpers/with-env.ts'; const tmpRoots: string[] = []; @@ -150,4 +150,48 @@ describe('gbrain takes CLI source scoping', () => { expect(added).toHaveLength(0); expect(existsSync(join(brainDir, 'shared/page.md'))).toBe(false); }); + + test('supersede looks up the active row it is replacing (#2663)', async () => { + const brainDir = mkdtempSync(join(tmpdir(), 'gbrain-takes-supersede-')); + const home = mkdtempSync(join(tmpdir(), 'gbrain-takes-home-')); + tmpRoots.push(brainDir, home); + const listCalls: TakesListOpts[] = []; + const engine = { + getConfig: async () => null, + executeRaw: async (sql: string, params: unknown[] = []) => { + if (sql.includes('FROM sources WHERE id = $1')) return [{ id: params[0] as string }]; + if (sql.includes('FROM sources WHERE local_path IS NOT NULL')) return []; + if (sql.includes('FROM pages WHERE slug = $1 AND source_id = $2')) return [{ id: 11 }]; + return []; + }, + listTakes: async (opts: TakesListOpts) => { + listCalls.push(opts); + return [{ + page_id: 11, + row_num: 3, + claim: 'Current claim', + kind: 'take', + holder: 'self', + weight: 0.8, + active: true, + }]; + }, + supersedeTake: async () => ({ oldRow: 3, newRow: 4 }), + } as unknown as BrainEngine; + + await withEnv({ GBRAIN_SOURCE: undefined, GBRAIN_HOME: home }, async () => { + await runTakes(engine, [ + 'supersede', + 'shared/page', + '--row', + '3', + '--claim', + 'Replacement claim', + '--dir', + brainDir, + ]); + }); + + expect(listCalls).toEqual([{ page_id: 11, active: true, limit: 500 }]); + }); }); From 5d7a8d7d4b7dc13b7d2e80925acd3e14e4ae2c40 Mon Sep 17 00:00:00 2001 From: Igby Date: Thu, 23 Jul 2026 18:13:07 -0600 Subject: [PATCH 09/39] chore(gitignore): ignore CLAUDE.local.md / AGENTS.local.md (#3290) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent tools (Claude Code, Codex, …) support a *.local.md counterpart to the committed CLAUDE.md/AGENTS.md for personal, per-clone instruction overrides that load after the committed file and are meant to stay uncommitted. gbrain already ignores other workspace-local agent artifacts (.context/, .claude/), so this fills the remaining gap for the two root-level override files. Explicit filenames rather than a *.local.md glob, matching the existing commented, specific style. --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index d0ad7dfad..ab59e82c5 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,11 @@ export/ # .context/test-shards/. Workspace-local by design — never committed. .context/ +# Local agent instruction overrides (CLAUDE.local.md / AGENTS.local.md) — personal, +# per-clone, loaded after the committed CLAUDE.md/AGENTS.md. Never committed. +CLAUDE.local.md +AGENTS.local.md + # Tier 3 PGLite snapshot fixture (built on demand by build:pglite-snapshot) test/fixtures/pglite-snapshot.tar test/fixtures/pglite-snapshot.version From 26b938c37dcbd768e1aa1aae4dbe2984c089ced8 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:28:22 -0700 Subject: [PATCH 10/39] fix(auth): expose OAuth source grants in whoami (takeover of #3279) (#3332) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebase of #3279 onto current master: the whoami oauth shape gains source_id (AuthInfo.sourceId, null when absent) and federated_read (AuthInfo.allowedSources, [] when absent) — read-only self-introspection that widens no grant. Re-applied against the post-#3091 description string (stdio transport shape preserved) and merged the grant tests into the current whoami.test.ts alongside the stdio cases. Co-authored-by: Garry Tan Co-authored-by: boundless-forest Co-authored-by: Claude Fable 5 --- docs/architecture/KEY_FILES.md | 2 +- src/core/operations.ts | 6 ++- test/whoami.test.ts | 69 +++++++++++++++++++++++++++++++--- 3 files changed, 69 insertions(+), 8 deletions(-) diff --git a/docs/architecture/KEY_FILES.md b/docs/architecture/KEY_FILES.md index bc3aaec0d..893726577 100644 --- a/docs/architecture/KEY_FILES.md +++ b/docs/architecture/KEY_FILES.md @@ -10,7 +10,7 @@ per-release `**vX.Y.Z:**` narration — CI enforces this - `src/commands/serve-http.ts` confidential revoke extension — a pre-router `/revoke` handler validates the RFC 7009 body, verifies hash-only secrets for both `client_secret_post` and `client_secret_basic`, rejects mixed authentication, preserves the SDK path for public clients, and separates opaque client-auth failures from retryable/backend failures. OAuth metadata advertises both confidential methods. Pinned by `test/e2e/serve-http-oauth.test.ts`. -- `src/core/operations.ts` — Contract-first operation definitions (the foundation). Exports upload validators `validateUploadPath`, `validatePageSlug`, `validateFilename`, plus `matchesSlugAllowList(slug, prefixes)` (glob matcher: `/*` matches recursive children; bare `` matches exact only). `OperationContext.remote` is a REQUIRED field flagging untrusted callers; `OperationContext.allowedSlugPrefixes` is the trusted-workspace allow-list set by the dream cycle; `OperationContext.auth?: AuthInfo` is threaded through HTTP dispatch for scope enforcement in `serve-http.ts` before the op runs. `enforceSubagentSlugFence(ctx, slug, opName)` is the shared fail-closed subagent write fence: when `viaSubagent` and `allowedSlugPrefixes` is set, the slug must match the allow-list; else the legacy `wiki/agents//...` namespace check applies. Both `put_page` and `add_timeline_entry` (subagent-allowlisted) route through it. Auto-link skipped only when `remote=true && !trustedWorkspace`. Every `Operation` carries `scope?: 'read' | 'write' | 'admin'` + `localOnly?: boolean`; `sync_brain`, `file_upload`, `file_list`, `file_url` are `admin + localOnly` (rejected over HTTP). Four trust-boundary call sites (`put_page` allowlist, file_upload trust-narrowing, submit_job protected-name guard, auto-link skip) use FAIL-CLOSED semantics: `ctx.remote === false` for trusted-only sites, `ctx.remote !== false` for "untrust unless explicit-false" — anything not strictly `false` is treated as remote (closes the HTTP MCP shell-job RCE where a read+write OAuth token could submit `shell` jobs). `sourceScopeOpts(ctx)` encodes the source-scoped read precedence ladder — federated array (`ctx.auth.allowedSources`) wins over scalar (`ctx.sourceId`/`ctx.auth.sourceId`) over nothing; every read-side op handler routes through it so a source-bound OAuth client can't see neighboring sources via `search`/`query`/`list_pages`/`get_page`/`find_experts`/query's image path, plus the by-slug reads `get_tags`/`get_links`/`get_backlinks`/`get_timeline` (and `get_page`'s tag fetch, which resolves against the concrete page's own `source_id`). `linkReadScopeOpts(ctx)` is the link-read sibling for `get_links`/`get_backlinks`: a link row references three pages (from/to/origin), and the engine's federated (`sourceIds[]`) branch scopes ALL THREE while its scalar (`sourceId`) branch scopes only the near endpoint (by design — trusted internal callers like `reconcileLinks` and back-link validators call the engine with a scalar scope and need the cross-source view). For an UNTRUSTED remote caller (`ctx.remote !== false`) carrying only a scalar scope it promotes that scope to a single-element `sourceIds:[id]`, routing them through the all-endpoint branch so a far/origin slug from outside the grant isn't disclosed; a federated array passes through unchanged and trusted local CLI (`ctx.remote === false`) keeps the scalar cross-source view. `thinkSourceScopeOpts(ctx)` maps the same precedence ladder onto `runThink`'s public options (`allowedSources`/`sourceId`) so the `think` op's gather and trajectory stages inherit the caller's source grant. `put_page`'s inline disk write-through is the shared `writePageThrough` helper (`src/core/write-through.ts`), ATOMIC via temp-sibling + rename so a crash or concurrent `gbrain sync` can't read a half-written `.md`; same helper backs `gbrain brainstorm/lsd --save`. Link provenance surface (#1941): `add_link` (`gbrain link`/`link-add`) + `remove_link` (`gbrain unlink`/`link-rm`) expose `link_source`/`link_type`; `add_link` rejects the reconciliation-managed built-ins via `MANAGED_LINK_SOURCES` (`markdown`/`frontmatter`/`mentions`/`wikilink-resolved`) and defaults omitted provenance to `'manual'` (the engine's own default stays `'markdown'` for internal callers); `list_link_sources` (`gbrain link-sources`, read) lists provenances via `sourceScopeOpts`. CLI aliases register through `cliHints.aliases` (collision-guarded in `src/cli.ts`). +- `src/core/operations.ts` — Contract-first operation definitions (the foundation). Exports upload validators `validateUploadPath`, `validatePageSlug`, `validateFilename`, plus `matchesSlugAllowList(slug, prefixes)` (glob matcher: `/*` matches recursive children; bare `` matches exact only). `OperationContext.remote` is a REQUIRED field flagging untrusted callers; `OperationContext.allowedSlugPrefixes` is the trusted-workspace allow-list set by the dream cycle; `OperationContext.auth?: AuthInfo` is threaded through HTTP dispatch for scope enforcement in `serve-http.ts` before the op runs. OAuth `whoami` exposes the authenticated `AuthInfo.sourceId` and `AuthInfo.allowedSources` grants as `source_id` and `federated_read`; absent grants serialize fail-closed as `null` and `[]`, while local, legacy, and stdio response shapes stay unchanged. `enforceSubagentSlugFence(ctx, slug, opName)` is the shared fail-closed subagent write fence: when `viaSubagent` and `allowedSlugPrefixes` is set, the slug must match the allow-list; else the legacy `wiki/agents//...` namespace check applies. Both `put_page` and `add_timeline_entry` (subagent-allowlisted) route through it. Auto-link skipped only when `remote=true && !trustedWorkspace`. Every `Operation` carries `scope?: 'read' | 'write' | 'admin'` + `localOnly?: boolean`; `sync_brain`, `file_upload`, `file_list`, `file_url` are `admin + localOnly` (rejected over HTTP). Four trust-boundary call sites (`put_page` allowlist, file_upload trust-narrowing, submit_job protected-name guard, auto-link skip) use FAIL-CLOSED semantics: `ctx.remote === false` for trusted-only sites, `ctx.remote !== false` for "untrust unless explicit-false" — anything not strictly `false` is treated as remote (closes the HTTP MCP shell-job RCE where a read+write OAuth token could submit `shell` jobs). `sourceScopeOpts(ctx)` encodes the source-scoped read precedence ladder — federated array (`ctx.auth.allowedSources`) wins over scalar (`ctx.sourceId`/`ctx.auth.sourceId`) over nothing; every read-side op handler routes through it so a source-bound OAuth client can't see neighboring sources via `search`/`query`/`list_pages`/`get_page`/`find_experts`/query's image path, plus the by-slug reads `get_tags`/`get_links`/`get_backlinks`/`get_timeline` (and `get_page`'s tag fetch, which resolves against the concrete page's own `source_id`). `linkReadScopeOpts(ctx)` is the link-read sibling for `get_links`/`get_backlinks`: a link row references three pages (from/to/origin), and the engine's federated (`sourceIds[]`) branch scopes ALL THREE while its scalar (`sourceId`) branch scopes only the near endpoint (by design — trusted internal callers like `reconcileLinks` and back-link validators call the engine with a scalar scope and need the cross-source view). For an UNTRUSTED remote caller (`ctx.remote !== false`) carrying only a scalar scope it promotes that scope to a single-element `sourceIds:[id]`, routing them through the all-endpoint branch so a far/origin slug from outside the grant isn't disclosed; a federated array passes through unchanged and trusted local CLI (`ctx.remote === false`) keeps the scalar cross-source view. `thinkSourceScopeOpts(ctx)` maps the same precedence ladder onto `runThink`'s public options (`allowedSources`/`sourceId`) so the `think` op's gather and trajectory stages inherit the caller's source grant. `put_page`'s inline disk write-through is the shared `writePageThrough` helper (`src/core/write-through.ts`), ATOMIC via temp-sibling + rename so a crash or concurrent `gbrain sync` can't read a half-written `.md`; same helper backs `gbrain brainstorm/lsd --save`. Link provenance surface (#1941): `add_link` (`gbrain link`/`link-add`) + `remove_link` (`gbrain unlink`/`link-rm`) expose `link_source`/`link_type`; `add_link` rejects the reconciliation-managed built-ins via `MANAGED_LINK_SOURCES` (`markdown`/`frontmatter`/`mentions`/`wikilink-resolved`) and defaults omitted provenance to `'manual'` (the engine's own default stays `'markdown'` for internal callers); `list_link_sources` (`gbrain link-sources`, read) lists provenances via `sourceScopeOpts`. CLI aliases register through `cliHints.aliases` (collision-guarded in `src/cli.ts`). - `src/core/engine.ts` — Pluggable engine interface (BrainEngine). `clampSearchLimit(limit, default, cap)` takes an explicit cap so per-operation caps can be tighter than `MAX_SEARCH_LIMIT`. Exports `LinkBatchInput`/`TimelineBatchInput` for the bulk-insert API (`addLinksBatch`/`addTimelineEntriesBatch`). `readonly kind: 'postgres' | 'pglite'` discriminator lets `src/core/migrate.ts` and others branch without `instanceof` + dynamic imports. Methods: `batchLoadEmotionalInputs(slugs?)` (CTE-shaped read with per-table aggregates so page × N tags × M takes never produces N×M rows), `setEmotionalWeightBatch(rows)` (`UPDATE FROM unnest($1::text[],$2::text[],$3::real[])` composite-keyed on `(slug, source_id)`), `getRecentSalience(opts)`, `findAnomalies(opts)`. `PageFilters` has `sort?: 'updated_desc'|'updated_asc'|'created_desc'|'slug'` + `PAGE_SORT_SQL` whitelist consumed by both engines. `listAllPageRefs(): Promise>` ordered by `(source_id, slug)` — cheap cross-source enumeration replacing the `getAllSlugs()→getPage(slug)` N+1 (which silently defaulted to `source_id='default'`); parity across postgres-engine.ts + pglite-engine.ts; Pinned by `test/e2e/multi-source-bug-class.test.ts`. `SearchOpts`+`PageFilters` add `sourceIds?: string[]` (federated read axis; both engines apply `WHERE source_id = ANY($N::text[])` when set, preserve scalar `sourceId` fast path when unset); `traverseGraph(slug, depth, opts?)` and `traversePaths(slug, opts?)` accept `opts.sourceId`/`opts.sourceIds`. The by-slug read methods carry the same federated axis: `getTags`/`getLinks`/`getBacklinks` opts and `TimelineOpts` (consumed by `getTimeline`) accept `sourceIds?: string[]` taking precedence over the scalar `sourceId` (`source_id = ANY($::text[])` scoping the slug→page-id lookup); the link reads (`getLinks`/`getBacklinks`) scope ALL THREE endpoints (from/to/origin) on the federated branch while the scalar branch scopes only the near endpoint for trusted internal cross-source callers. `traverseGraph` opts has `frontierCap?: number` (per-iteration recursive-CTE cap, approx per-BFS-layer); return type `Promise` for MCP wire stability; export `TraverseGraphOpts`; Postgres uses parenthesized `LIMIT N ORDER BY (slug, id)` inside the recursive term, PGLite mirrors with positional params; Pinned by `test/regressions/v0_36_frontier_cap.test.ts`. Phantom-redirect methods: `refreshPageBody(slug, sourceId, compiled_truth, timeline, content_hash)` narrow-UPDATEs three columns + updated_at, skipping soft-deleted rows (content_hash refresh required so `gbrain sync` sees the canonical as unchanged after fence merge); `migrateFactsToCanonical(phantomSlug, canonicalSlug, sourceId)` UPDATEs `entity_slug`+`source_markdown_slug` on every active fact row keyed on the phantom, preserving embedding/validUntil/kind/status/source_session/confidence; parity at `test/phantom-redirect-engine-parity.test.ts`. `getAdjacencyBoosts(pageIds): Promise>` powers the per-query graph-signals stage — one SQL query returning inbound-link counts among top-K plus a cross-source count (links from differing `source_id`); `COALESCE(p.source_id,'default')` null safety, `HAVING >= 1`, cross-source CASE-WHEN excludes the target's own source; parity SQL across both engines; `SearchResult` gains optional `base_score`, `backlink_boost`, `salience_boost`, `recency_boost`, `exact_match_boost`, `graph_adjacency_boost`, `graph_cross_source_boost`, `session_demote_factor`, `reranker_delta` + internal staging fields; Pinned by `test/e2e/graph-signals-engine.test.ts`. Two REQUIRED methods: `deletePages(slugs, {sourceId}): Promise` (single-batch primitive returning slugs actually deleted) and `resolveSlugsByPaths(paths, {sourceId}): Promise>` (batch path→slug lookup); `sourceId` REQUIRED on both at the type level (asymmetric with single-row `deletePage` which keeps optional/'default'); both short-circuit on empty input and throw when `> DELETE_BATCH_SIZE`. Embedding-signature stale-detection quartet: `countStaleChunks(opts?)` gains optional `signature?: string` widening the stale predicate from `embedding IS NULL` to ALSO include chunks whose JOINed page `embedding_signature IS NOT NULL AND <> $signature` (NULL signature is GRANDFATHERED, never counted; omit `signature` for the legacy NULL-only count); `sumStaleChunkChars(opts?: {sourceId?, signature?}): Promise` = `SUM(LENGTH(chunk_text))` over stale chunks (same predicate + embed_skip filter + optional sourceId scope), used by `gbrain sync --all` cost preview via `estimateCostFromChars`; `setPageEmbeddingSignature(slug, {sourceId?, signature})` stamps `pages.embedding_signature` after a page's chunks (re)embed, idempotent no-op when page absent; `invalidateStaleSignatureEmbeddings({signature, sourceId?}): Promise` NULLs `embedding`+`embedded_at` on every chunk whose page signature is set AND differs, returning the count, called BEFORE `listStaleChunks` so signature-drift pages flow through the NULL-embedding keyset cursor unchanged (NULL never invalidated). Widens `findOrphanPages(opts?: {sourceId?, sourceIds?})` (candidate-side scoping only; inbound links counted from any source). Pinned by `test/sum-stale-chunk-chars.test.ts`, `test/embedding-signature-stale.test.ts`, `test/e2e/engine-parity.test.ts`. Free-text alias layer: `resolveAliases(aliasNorms, opts?): Promise>>` (READ; maps each normalized alias to declaring `(slug, source_id)` pairs, source-scoped) and `setPageAliases(slug, sourceId, aliasNorms)` (WRITE; replaces the full alias set, delete-then-insert, empty clears, idempotent on the unique triple), called by the `importFromContent` ingest projection and the `reindex --aliases` backfill; parity across both engines, Pinned by `test/search/page-aliases-engine.test.ts`. `searchVector` in both engines injects the shared `buildBestPerPagePoolCte` per-page max-pool so a page surfaces on its strongest chunk. `executeRawDirect(sql, params?, opts?)` is the lock-hot-path sibling of `executeRaw`: same single-statement contract, but routes to the direct session-mode pool when dual-pool is active (Postgres/Supabase port 5432) so a long-held lock heartbeat survives the transaction pooler's per-transaction connection recycling; PGLite delegates straight to `executeRaw` (no pooler). Both engines implement it; the Minion lock path (`claim`/`renewLock`) is the consumer. `reconnect(ctx?: {error?})` is a REQUIRED lifecycle method on BOTH engines: it recovers a dropped connection using the config captured at the last `connect()`, so callers (autopilot health probe, `batchRetry`) never `disconnect()` + bare `connect()` (which loses the config and throws `database_url undefined` forever, and opens a null-connection window). PostgresEngine rebuilds its pool with a `_reconnecting` reentrancy guard and emits pool-recovery audit; PGLite is single-writer in-process so it just restores the saved data dir for parity. - `src/core/engine-constants.ts` — single source of truth for engine batch-sizing constants. Exports `DELETE_BATCH_SIZE = 500` consumed by both engines' `deletePages` + `resolveSlugsByPaths` and by the sync delete + rename loops. Lives outside `engine.ts` (the interface module) to avoid circular-import worry — bounded per-statement work for predictable lock hold time + write amplification. - `src/core/background-work.ts` (#1762/#1745/#1775) — process background-work registry: the single owner of "drain every fire-and-forget DB-write sink before the CLI disconnects." `registerBackgroundWorkDrainer({name, order, drain(timeoutMs), abort?})` + `drainAllBackgroundWorkForCliExit({timeoutMs})` over a `Map` (idempotent registration by name; `__registerDrainerForTest` returns an unregister handle). Drains in explicit `(order, name)` order — facts FIRST (order 0) so its abort-path DB `logIngest` runs against the freshest live engine — and AWAITS `abort()` only when `drain()` reports `unfinished>0`. Best-effort per drainer: one sink's failure never blocks the others or the disconnect. FIVE sinks register at module import: `facts/queue.ts` (order 0; `abort`=`shutdown()` cancels a hung facts:absorb Haiku via internalAbort), `last-retrieved.ts` (order 1), `search/hybrid.ts` (order 2; `awaitPendingSearchCacheWrites` bounded via `Promise.race`), `eval-capture.ts` (order 3; `captureEvalCandidate` self-tracks its promise via `awaitPendingEvalCaptures`), `context/volunteer-events.ts` (order 4, #2095 — batched volunteer-event INSERTs, drained like the rest). Every cli.ts teardown site reaches it through `finishCliTeardown` (`src/core/cli-force-exit.ts`), which drains the registry before `engine.disconnect()` — closing the PGLite busy-loop where `db.close()` raced an in-flight job and pinned the single-writer lock (#1762). Exports `backgroundWorkSinkCount()` so the teardown helper computes its backstop deadline from the registered sink count. CLI-EXIT-ONLY: the facts `shutdown()` abort is permanent process state, never call in a long-lived `gbrain serve`. Companion changes: `src/core/ai/gateway.ts` `withDefaultTimeout(caller, ms)` bounds every outbound AI call (chat 300s, embed+multimodal 60s; env `GBRAIN_AI_{CHAT,EMBED,MULTIMODAL}_TIMEOUT_MS`; composed with caller signals via `AbortSignal.any`) and the teardown backstop honors an errored op's exit code so a hung disconnect can't mask failure as success (see `cli-force-exit.ts`); `src/core/postgres-engine.ts` `reconnect()` module-mode branch re-establishes via idempotent `db.connect()` + `connectionManager.setReadPool` refresh instead of `db.disconnect()` (no null window for concurrent ops; fail-loud on real connect failure — #1745); `src/core/search/hybrid.ts` `embedQueryBounded` + a shared `QueryEmbedDeadline` (6s, floored 2s per embed via `MIN_QUERY_EMBED_BUDGET_MS`; env `GBRAIN_QUERY_EMBED_TIMEOUT_MS`) bounds the cache-lookup AND inner query embeds so a stalled provider falls back to keyword instead of stalling the whole op (#1775). Incorporates + hardens PR #1763 (@ElliotDrel). Pinned by `test/core/background-work.test.ts`, `test/search/query-embed-deadline.test.ts`, `test/eval-capture-drain.test.ts`, `test/e2e/postgres-reconnect-singleton.test.ts`, `test/e2e/pglite-cli-exit.serial.test.ts`, `test/fix-wave-structural.test.ts`. diff --git a/src/core/operations.ts b/src/core/operations.ts index 670d066da..4397a421a 100644 --- a/src/core/operations.ts +++ b/src/core/operations.ts @@ -3823,7 +3823,7 @@ const whoami: Operation = { name: 'whoami', description: 'Introspect the calling identity. Returns one of three transport shapes: ' + - '{transport: "oauth", client_id, client_name, scopes, expires_at}, ' + + '{transport: "oauth", client_id, client_name, scopes, expires_at, source_id, federated_read}, ' + '{transport: "legacy", token_name, scopes, expires_at: null}, or ' + '{transport: "local", scopes: []}, or {transport: "stdio", scopes: []} ' + 'for the auth-less stdio MCP pipe. Throws unknown_transport when the ' + @@ -3865,6 +3865,10 @@ const whoami: Operation = { client_name: ctx.auth.clientName ?? ctx.auth.clientId, scopes: ctx.auth.scopes, expires_at: ctx.auth.expiresAt ?? null, + // Read-only self-introspection of the token's source grants — + // widens nothing; absent grants serialize fail-closed (null / []). + source_id: ctx.auth.sourceId ?? null, + federated_read: ctx.auth.allowedSources ?? [], }; } return { diff --git a/test/whoami.test.ts b/test/whoami.test.ts index 73aef6171..28f765bcd 100644 --- a/test/whoami.test.ts +++ b/test/whoami.test.ts @@ -55,23 +55,75 @@ describe('whoami op contract', () => { expect(result.scopes).toEqual([]); }); - test('oauth transport returns full client identity', async () => { + test('oauth transport returns client identity and exact source grants', async () => { const auth: AuthInfo = { token: 'gbrain_at_xxx', clientId: 'gbrain_cl_abc', clientName: 'gstack-test', scopes: ['read', 'sources_admin'], expiresAt: 1234567890, + sourceId: 'hot-memory', + allowedSources: ['hot-memory', 'canonical-brain'], + }; + const result = (await whoami.handler( + ctxWith({ remote: true, sourceId: 'transport-fallback', auth }), + {}, + )) as any; + expect(result).toEqual({ + transport: 'oauth', + client_id: 'gbrain_cl_abc', + client_name: 'gstack-test', + scopes: ['read', 'sources_admin'], + expires_at: 1234567890, + source_id: 'hot-memory', + federated_read: ['hot-memory', 'canonical-brain'], + }); + }); + + test('oauth transport uses fail-closed empty values when source grants are absent', async () => { + const auth: AuthInfo = { + token: 'gbrain_at_pre_migration', + clientId: 'gbrain_cl_pre_migration', + scopes: ['read'], + }; + const result = (await whoami.handler( + ctxWith({ remote: true, sourceId: 'transport-fallback', auth }), + {}, + )) as any; + expect(result.source_id).toBeNull(); + expect(result.federated_read).toEqual([]); + }); + + test('oauth transport preserves an explicit empty federated grant', async () => { + const auth: AuthInfo = { + token: 'gbrain_at_empty', + clientId: 'gbrain_cl_empty', + scopes: ['read', 'write'], + sourceId: 'hot-memory', + allowedSources: [], }; const result = (await whoami.handler( ctxWith({ remote: true, auth }), {}, )) as any; - expect(result.transport).toBe('oauth'); - expect(result.client_id).toBe('gbrain_cl_abc'); - expect(result.client_name).toBe('gstack-test'); - expect(result.scopes).toEqual(['read', 'sources_admin']); - expect(result.expires_at).toBe(1234567890); + expect(result.source_id).toBe('hot-memory'); + expect(result.federated_read).toEqual([]); + }); + + test('oauth transport does not widen federated_read with the write source', async () => { + const auth: AuthInfo = { + token: 'gbrain_at_narrow', + clientId: 'gbrain_cl_narrow', + scopes: ['read', 'write'], + sourceId: 'hot-memory', + allowedSources: ['canonical-brain'], + }; + const result = (await whoami.handler( + ctxWith({ remote: true, auth }), + {}, + )) as any; + expect(result.source_id).toBe('hot-memory'); + expect(result.federated_read).toEqual(['canonical-brain']); }); test('legacy transport (token name as clientId, no gbrain_cl_ prefix)', async () => { @@ -150,6 +202,11 @@ describe('whoami op contract', () => { }); describe('whoami op metadata', () => { + test('description documents OAuth source grant fields', () => { + expect(whoami.description).toContain('source_id'); + expect(whoami.description).toContain('federated_read'); + }); + test('scope is read (any authenticated caller can introspect itself)', () => { expect(whoami.scope).toBe('read'); }); From ef1133df3e25ffabe520d6e41ceb9d83037b0809 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:28:29 -0700 Subject: [PATCH 11/39] docs(security): Docker network isolation for co-located self-hosted Postgres (#3270) (#3331) OAuth/source scoping only guards the serve --http path; a container sharing Docker's default bridge with the brain's Postgres can open a direct DB session without a token. Adds a 'Co-located Docker workloads' subsection to docs/mcp/DEPLOY.md with the operator checklist, a trust-boundary paragraph in SECURITY.md, and an ops note + cross-link in the company-brain tutorial. Fixes #3270 Co-authored-by: Garry Tan Co-authored-by: Claude Fable 5 --- SECURITY.md | 12 +++++++++++ docs/mcp/DEPLOY.md | 37 +++++++++++++++++++++++++++++++++ docs/tutorials/company-brain.md | 4 ++++ llms-full.txt | 37 +++++++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+) diff --git a/SECURITY.md b/SECURITY.md index 60def9409..833809205 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -135,6 +135,18 @@ the PGLite schema. Local agents continue to use stdio (`gbrain serve`). Running `--http` against a PGLite-backed install fails fast with a clear error message at startup. +### Docker network isolation (self-hosted Postgres) + +OAuth and source scoping enforce isolation on the `serve --http` path only. +Raw Postgres reachability bypasses both: a container that shares Docker's +default `bridge` network with the brain's Postgres can open a direct DB +session without any token and read every source. Put the brain's Postgres on +a user-defined Docker network with nothing untrusted on it, publish its port +loopback-only (if at all), and never put `DATABASE_URL` or a Postgres +password in untrusted agent containers — those should reach the brain +exclusively via OAuth against `serve --http`. Full operator checklist: +[docs/mcp/DEPLOY.md — Co-located Docker workloads](docs/mcp/DEPLOY.md#co-located-docker-workloads-self-hosted-postgres). + ### CORS Default-deny: no `Access-Control-Allow-Origin` header is sent unless an diff --git a/docs/mcp/DEPLOY.md b/docs/mcp/DEPLOY.md index 62bd84156..e4182d593 100644 --- a/docs/mcp/DEPLOY.md +++ b/docs/mcp/DEPLOY.md @@ -258,6 +258,43 @@ the user owns the machine. See [ALTERNATIVES.md](ALTERNATIVES.md) for a comparison of ngrok, Tailscale Funnel, and cloud hosts (Fly.io, Railway). +### Co-located Docker workloads (self-hosted Postgres) + +OAuth scopes and source scoping guard the `gbrain serve --http` path. They do +NOT guard raw Postgres. If the brain's Postgres runs as a container on the same +Docker host as other workloads (agent runtimes, n8n, staging fixtures), any +container sharing Docker's default `bridge` network can open a direct DB +session — no OAuth token required — and read every source. That silently +recreates a privileged path underneath the isolation you configured at the MCP +layer. + +Network-zone the host so untrusted containers can never reach Postgres: + +``` +Docker host +├── gbrain-net ← ONLY the brain's Postgres (+ gbrain serve, if containerized) +├── agent--net ← each untrusted agent runtime, isolated +└── default bridge ← no secret-bearing databases +``` + +Operator checklist: + +```text +[ ] Postgres is on a user-defined Docker network, not the default bridge + (or nothing else runs on that bridge) +[ ] If Postgres publishes a host port at all, it binds loopback only + (`-p 127.0.0.1:5432:5432`, never `0.0.0.0`) +[ ] Untrusted agent containers have no DATABASE_URL or Postgres password +[ ] Untrusted agents reach the brain via OAuth/Bearer against serve --http only + (host loopback via host.docker.internal / host gateway — never gbrain-net) +[ ] OAuth clients are least-privilege: scoped --source / --federated-read, + pre-minted short-lived tokens preferred over long-lived client secrets +[ ] Isolation verified: a team-scoped client cannot read internal-only sources +``` + +Optional defense-in-depth: a dedicated Postgres role (or RLS) limited to the +allowed `source_id`s, so even a leaked connection string can't read everything. + ## Troubleshooting **"missing_auth" error** diff --git a/docs/tutorials/company-brain.md b/docs/tutorials/company-brain.md index b2dc98b00..6ebe643e1 100644 --- a/docs/tutorials/company-brain.md +++ b/docs/tutorials/company-brain.md @@ -484,6 +484,10 @@ Returns a per-source dashboard: when each source last synced, how many pages, ho The admin dashboard at `https://brain.acme-co.com/admin` shows live request volume, registered OAuth clients, recent activity, and brain stats. Use the admin bootstrap token from Part 4 to log in the first time, then register additional admin users from inside the dashboard. +### If agents run as containers on the same Docker host + +OAuth source scoping only guards the HTTP MCP path. If the brain's Postgres and your teammates' agent runtimes are containers on the same Docker host, make sure the agents can't reach Postgres directly over Docker's default bridge network — a direct DB session skips OAuth entirely. Put Postgres on its own user-defined network, publish it loopback-only if at all, and never hand agent containers a `DATABASE_URL`. The copy-paste operator checklist lives in [docs/mcp/DEPLOY.md — Co-located Docker workloads](../mcp/DEPLOY.md#co-located-docker-workloads-self-hosted-postgres). + --- ## Part 13: Cost and speed expectations diff --git a/llms-full.txt b/llms-full.txt index 890294b93..0184bd797 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -3905,6 +3905,43 @@ the user owns the machine. See [ALTERNATIVES.md](ALTERNATIVES.md) for a comparison of ngrok, Tailscale Funnel, and cloud hosts (Fly.io, Railway). +### Co-located Docker workloads (self-hosted Postgres) + +OAuth scopes and source scoping guard the `gbrain serve --http` path. They do +NOT guard raw Postgres. If the brain's Postgres runs as a container on the same +Docker host as other workloads (agent runtimes, n8n, staging fixtures), any +container sharing Docker's default `bridge` network can open a direct DB +session — no OAuth token required — and read every source. That silently +recreates a privileged path underneath the isolation you configured at the MCP +layer. + +Network-zone the host so untrusted containers can never reach Postgres: + +``` +Docker host +├── gbrain-net ← ONLY the brain's Postgres (+ gbrain serve, if containerized) +├── agent--net ← each untrusted agent runtime, isolated +└── default bridge ← no secret-bearing databases +``` + +Operator checklist: + +```text +[ ] Postgres is on a user-defined Docker network, not the default bridge + (or nothing else runs on that bridge) +[ ] If Postgres publishes a host port at all, it binds loopback only + (`-p 127.0.0.1:5432:5432`, never `0.0.0.0`) +[ ] Untrusted agent containers have no DATABASE_URL or Postgres password +[ ] Untrusted agents reach the brain via OAuth/Bearer against serve --http only + (host loopback via host.docker.internal / host gateway — never gbrain-net) +[ ] OAuth clients are least-privilege: scoped --source / --federated-read, + pre-minted short-lived tokens preferred over long-lived client secrets +[ ] Isolation verified: a team-scoped client cannot read internal-only sources +``` + +Optional defense-in-depth: a dedicated Postgres role (or RLS) limited to the +allowed `source_id`s, so even a leaked connection string can't read everything. + ## Troubleshooting **"missing_auth" error** From cd18081f4ae984acabf28a3444f98436eb460145 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:28:33 -0700 Subject: [PATCH 12/39] fix(takes): keyword search matches words in long claims via word_similarity (#3267) (#3333) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both engines' searchTakes used whole-string trigram similarity (claim % query), which structurally cannot pass the 0.3 threshold for a short keyword against a 100-200 char claim — keyword search returned zero results on real brains. Switch the predicate to word similarity (query <% claim) and rank by word_similarity(query, claim), in both postgres-engine and pglite-engine per the engine-parity invariant. Holder allow-list and source-scope filters unchanged. Regression test: single-word query must match a long claim containing it (fails under the old predicate). Fixes #3267 Co-authored-by: Garry Tan Co-authored-by: Claude Fable 5 --- src/core/pglite-engine.ts | 4 ++-- src/core/postgres-engine.ts | 4 ++-- test/takes-engine.test.ts | 16 ++++++++++++++++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/core/pglite-engine.ts b/src/core/pglite-engine.ts index 3b6e422c9..c29b82887 100644 --- a/src/core/pglite-engine.ts +++ b/src/core/pglite-engine.ts @@ -4835,11 +4835,11 @@ export class PGLiteEngine implements BrainEngine { const { rows } = await this.db.query( `SELECT t.id AS take_id, t.page_id, p.slug AS page_slug, t.row_num, t.claim, t.kind, t.holder, t.weight, - similarity(t.claim, $1)::real AS score + word_similarity($1, t.claim)::real AS score FROM takes t JOIN pages p ON p.id = t.page_id WHERE t.active - AND t.claim % $1 + AND $1 <% t.claim AND ($2::text[] IS NULL OR t.holder = ANY($2::text[])) AND ($4::text[] IS NULL OR p.source_id = ANY($4::text[])) AND ($5::text IS NULL OR p.source_id = $5::text) diff --git a/src/core/postgres-engine.ts b/src/core/postgres-engine.ts index b86292b2a..55dbf9aea 100644 --- a/src/core/postgres-engine.ts +++ b/src/core/postgres-engine.ts @@ -4962,11 +4962,11 @@ export class PostgresEngine implements BrainEngine { const rows = await sql` SELECT t.id AS take_id, t.page_id, p.slug AS page_slug, t.row_num, t.claim, t.kind, t.holder, t.weight, - similarity(t.claim, ${query})::real AS score + word_similarity(${query}, t.claim)::real AS score FROM takes t JOIN pages p ON p.id = t.page_id WHERE t.active - AND t.claim % ${query} + AND ${query} <% t.claim AND ( ${opts.takesHoldersAllowList ?? null}::text[] IS NULL OR t.holder = ANY(${opts.takesHoldersAllowList ?? null}::text[]) diff --git a/test/takes-engine.test.ts b/test/takes-engine.test.ts index 826cf41ae..8bf1bbe8b 100644 --- a/test/takes-engine.test.ts +++ b/test/takes-engine.test.ts @@ -100,6 +100,22 @@ describe('searchTakes', () => { const worldHits = await engine.searchTakes('founder', { takesHoldersAllowList: ['world'] }); expect(worldHits.every(h => h.holder === 'world')).toBe(true); }); + + // #3267: whole-string trigram % structurally can't match a short keyword + // against a long claim (similarity between the full strings stays under the + // 0.3 threshold). word_similarity (<%) matches the keyword against the + // best-matching word span instead. + test('single-word keyword matches a long claim containing it (#3267)', async () => { + await engine.addTakesBatch([ + { + page_id: acmePageId, row_num: 50, + claim: 'Acme will consolidate the mid-market vertical SaaS landscape through disciplined acquisitions and a shared billing platform over the next five years', + kind: 'bet', holder: 'garry', weight: 0.6, + }, + ]); + const hits = await engine.searchTakes('consolidate'); + expect(hits.some(h => h.claim.includes('consolidate the mid-market'))).toBe(true); + }); }); describe('updateTake', () => { From 4b38724aa2d027994480b8482c18fe6a93785f40 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:28:38 -0700 Subject: [PATCH 13/39] fix(sources): federated-source pages visible to get_page/list_pages/resolve_slugs and no-grant MCP callers (#3242) (#3301) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pages ingested into a config.federated=true source were invisible to normal reads: get_page/list_pages scoped to the scalar resolved source ('default'), while the fully UNSCOPED resolve_slugs leaked every source's slugs — the reporter's exact observation matrix. - federatedSearchScope now backs get_page, list_pages and resolve_slugs (not just search/query), so the unqualified read surface shares one visibility set: grant > federated set > scalar source. resolve_slugs gains the missing sourceScopeOpts-family scoping (leak sealed). - The widening gate is now field-presence instead of ctx.remote: localFederatedSourceIds is populated only by server-side transports (never from caller params), so trust stays fail-closed while the stdio MCP transport (no GBRAIN_SOURCE) and the legacy HTTP token path (no operator-set permissions.source_id grant) can opt their unqualified callers into the operator-configured federated set. Tokens WITH a grant, per-call source_id, and OAuth allowedSources all still win and never widen. - gbrain sync now attributes its ingest-log row to the synced source instead of the shared 'default' bucket (attribution sub-bug). No engine SQL changes: getPage/listPages/resolveSlugs already accept sourceIds[] in both engines (#1393/#876). Fixes #3242 Co-authored-by: Garry Tan Co-authored-by: Claude Fable 5 --- src/commands/sync.ts | 3 + src/core/operations.ts | 69 ++++++++++++--------- src/mcp/dispatch.ts | 8 +++ src/mcp/http-transport.ts | 22 +++++++ src/mcp/server.ts | 15 +++++ test/local-federated-search-scope.test.ts | 73 +++++++++++++++++++++-- 6 files changed, 157 insertions(+), 33 deletions(-) diff --git a/src/commands/sync.ts b/src/commands/sync.ts index 377aaa033..28c359441 100644 --- a/src/commands/sync.ts +++ b/src/commands/sync.ts @@ -3384,6 +3384,9 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise { - return ctx.engine.resolveSlugs(p.partial as string); + // #3242: was fully UNSCOPED — the one read that leaked every source's + // slugs to any caller (the reporter's "resolve_slugs sees them but + // get_page doesn't" matrix). Route through the same visibility set as + // get_page/search: grant > federated set > scalar source. + return ctx.engine.resolveSlugs(p.partial as string, federatedSearchScope(ctx)); }, scope: 'read', }; diff --git a/src/mcp/dispatch.ts b/src/mcp/dispatch.ts index a37e23c92..18552fc80 100644 --- a/src/mcp/dispatch.ts +++ b/src/mcp/dispatch.ts @@ -54,6 +54,13 @@ export interface DispatchOpts { * resolves it from the per-token allow-list (eE3). */ sourceId?: string; + /** + * #3242: federated read set for callers with NO explicit source scope + * (stdio without GBRAIN_SOURCE; legacy HTTP tokens without an operator-set + * `permissions.source_id` grant). Transport-computed, never derived from + * caller params. See OperationContext.localFederatedSourceIds. + */ + localFederatedSourceIds?: string[]; /** * v0.31 (eD3): hook called by the dispatcher AFTER op.handler succeeds * to compute `_meta.brain_hot_memory` for the response. Wrapped in its @@ -216,6 +223,7 @@ export function buildOperationContext( // CLI / HTTP / stdio transports SHOULD pass an explicit sourceId via opts; // this fallback covers code paths that historically passed undefined. sourceId: opts.sourceId ?? 'default', + ...(opts.localFederatedSourceIds ? { localFederatedSourceIds: opts.localFederatedSourceIds } : {}), auth: opts.auth, }; } diff --git a/src/mcp/http-transport.ts b/src/mcp/http-transport.ts index c3ff28c27..b1ee57a5a 100644 --- a/src/mcp/http-transport.ts +++ b/src/mcp/http-transport.ts @@ -84,6 +84,14 @@ interface AuthResult { * Bounded to the stored grant — never widened to "all". */ auth?: AuthInfo; + /** + * #3242: true when the token row carries an operator-set + * `permissions.source_id` (string OR array — even a malformed one, which + * fails closed to 'default' without widening). false = the historical + * no-grant 'default' floor; ONLY that case gets the federated read set + * (config.federated sources) threaded as localFederatedSourceIds. + */ + hasSourceGrant?: boolean; } /* Legacy token source-scope parsing lives in core/legacy-token-scope.ts and is @@ -229,6 +237,9 @@ export async function startHttpTransport(opts: HttpTransportOptions) { // source unless the token carries an explicit grant (#1336 above). sourceId, auth, + // #3242: distinguish "operator granted a scope" from "historical + // no-grant floor" — only the latter widens to federated sources. + hasSourceGrant: perms?.source_id != null, }; } catch { return { ok: false }; @@ -379,10 +390,21 @@ export async function startHttpTransport(opts: HttpTransportOptions) { // takes_search / query (when it returns takes) can server-side filter. // v0.34.1 (#861): thread source-isolation scope. Legacy access_tokens // path defaults to 'default' per AuthResult.sourceId above. + // #3242: a token with NO operator-set source grant reads across the + // federated set (config.federated sources), not just the scalar + // 'default' floor. Granted tokens (hasSourceGrant) never widen. + let localFederated: string[] | undefined; + if (auth.hasSourceGrant === false && auth.sourceId) { + try { + const { localFederatedSourceIds } = await import('../core/source-resolver.ts'); + localFederated = await localFederatedSourceIds(engine, auth.sourceId, 'seed_default'); + } catch { /* scalar scope stands */ } + } const result = await dispatchToolCall(engine, toolName, args, { remote: true, takesHoldersAllowList: auth.takesHoldersAllowList, sourceId: auth.sourceId, + ...(localFederated ? { localFederatedSourceIds: localFederated } : {}), // #1336: thread the token's federated_read grant so read ops scope // to the operator-granted sources via sourceScopeOpts. auth: auth.auth, diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 6f98b01dc..425624e98 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -35,6 +35,20 @@ export async function startMcpServer(engine: BrainEngine) { // shape and cast through `any` (the SDK accepts it via the ServerResult union). server.setRequestHandler(CallToolRequestSchema, async (request: any): Promise => { const { name, arguments: params } = request.params; + // #3242: when the operator didn't pin a source via GBRAIN_SOURCE, stdio + // reads span every `config.federated = true` source (same visibility set + // as unqualified local CLI reads). GBRAIN_SOURCE set = explicit scope, + // no widening. Best-effort: a resolver failure keeps the scalar scope. + // ponytail: one tiny SELECT per tool call; cache it if it ever shows up. + let localFederated: string[] | undefined; + try { + const { localFederatedSourceIds } = await import('../core/source-resolver.ts'); + localFederated = await localFederatedSourceIds( + engine, + process.env.GBRAIN_SOURCE || 'default', + process.env.GBRAIN_SOURCE ? 'env' : 'seed_default', + ); + } catch { /* scalar scope stands */ } // v0.28: stdio MCP has no per-token auth (local pipe). Default the // takes-holder allow-list to ['world'] so agent-facing callers don't // see private hunches via takes_list / takes_search / query. Operators @@ -51,6 +65,7 @@ export async function startMcpServer(engine: BrainEngine) { // Operators who want a different source on stdio MCP should set // GBRAIN_SOURCE in the env or use --source via `gbrain call`. sourceId: process.env.GBRAIN_SOURCE || 'default', + ...(localFederated ? { localFederatedSourceIds: localFederated } : {}), // v0.31 (eD3): _meta.brain_hot_memory injection so Claude Desktop / // Code see the brain's relevant hot memory automatically alongside // every tool-call response. Best-effort; absorbs errors. diff --git a/test/local-federated-search-scope.test.ts b/test/local-federated-search-scope.test.ts index fc811a4f3..2a7dd1b64 100644 --- a/test/local-federated-search-scope.test.ts +++ b/test/local-federated-search-scope.test.ts @@ -11,9 +11,17 @@ * Fix: the CLI context builder computes `ctx.localFederatedSourceIds` * (resolved source + every other federated source) whenever the source * resolved via a NON-explicit tier; `federatedSearchScope` widens the scalar - * scope to that set for the `search` / `query` ops — trusted-local only - * (`ctx.remote === false`), never for remote callers, never when a per-call - * `source_id` or an explicit --source/env/dotfile was given. + * scope to that set — never when a per-call `source_id`, a grant array, or an + * explicit --source/env/dotfile was given. + * + * #3242 extends the same visibility set to `get_page` / `list_pages` / + * `resolve_slugs` (pages ingested into a `federated: true` source were + * invisible to normal reads while the unscoped resolve_slugs leaked them), + * and to transports whose caller carries NO explicit source scope (stdio + * without GBRAIN_SOURCE; legacy HTTP tokens without a `permissions.source_id` + * grant) — those transports now populate `localFederatedSourceIds` themselves, + * so the widening gate is field-presence (transport-decided, never + * param-controlled), not `ctx.remote`. */ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { PGLiteEngine } from '../src/core/pglite-engine.ts'; @@ -105,11 +113,19 @@ describe('federatedSearchScope — trust + explicitness matrix', () => { expect(federatedSearchScope(ctx)).toEqual({ sourceIds: ['default', 'wiki'] }); }); - test('remote caller NEVER widens (fail-closed), even if the field is set', () => { - const ctx = ctxOf({ remote: true, localFederatedSourceIds: ['default', 'wiki'] }); + test('remote caller WITHOUT the field never widens (fail-closed)', () => { + const ctx = ctxOf({ remote: true }); expect(federatedSearchScope(ctx)).toEqual({ sourceId: 'default' }); }); + test('#3242: remote caller widens when its transport populated the field (no-grant floor)', () => { + // The field is set only by server-side transports (stdio without + // GBRAIN_SOURCE / legacy HTTP token without a source grant) — never from + // caller params — so presence of the field IS the trust decision. + const ctx = ctxOf({ remote: true, localFederatedSourceIds: ['default', 'wiki'] }); + expect(federatedSearchScope(ctx)).toEqual({ sourceIds: ['default', 'wiki'] }); + }); + test('per-call source_id wins over the federated set', () => { const ctx = ctxOf({ localFederatedSourceIds: ['default', 'wiki'] }); expect(federatedSearchScope(ctx, 'wiki')).toEqual({ sourceId: 'wiki' }); @@ -152,3 +168,50 @@ describe('search op — unqualified local search spans federated sources', () => expect(slugs).toEqual(['notes/home']); }); }); + +// #3242 — pages in a federated source must be visible to the normal read ops, +// not just search/query; and resolve_slugs must be SCOPED (pre-fix it was the +// one read that leaked every source's slugs). +describe('#3242 — get_page / list_pages / resolve_slugs share the federated visibility set', () => { + const getPage = operations.find((o) => o.name === 'get_page')!; + const listPages = operations.find((o) => o.name === 'list_pages')!; + const resolveSlugsOp = operations.find((o) => o.name === 'resolve_slugs')!; + + function federatedCtx(overrides: Partial = {}): OperationContext { + return ctxOf({ localFederatedSourceIds: ['default', 'wiki'], ...overrides }); + } + + test('get_page: federated-source page readable on an unqualified ctx (pre-fix: page_not_found)', async () => { + const page = (await getPage.handler(federatedCtx(), { slug: 'wiki/topic' })) as { slug: string }; + expect(page.slug).toBe('wiki/topic'); + }); + + test('get_page: non-federated source stays invisible', async () => { + await expect(getPage.handler(federatedCtx(), { slug: 'private/topic' })).rejects.toThrow(/not found/i); + }); + + test('get_page: scalar ctx (explicit source, no field) keeps single-source behavior', async () => { + await expect(getPage.handler(ctxOf(), { slug: 'wiki/topic' })).rejects.toThrow(/not found/i); + }); + + test('list_pages: federated-source pages listed on an unqualified ctx (pre-fix: missing)', async () => { + const rows = (await listPages.handler(federatedCtx(), {})) as Array<{ slug: string }>; + const slugs = rows.map((r) => r.slug); + expect(slugs).toContain('notes/home'); + expect(slugs).toContain('wiki/topic'); + expect(slugs).not.toContain('private/topic'); + expect(slugs).not.toContain('old/topic'); + }); + + test('resolve_slugs: scoped to the visibility set (pre-fix: leaked every source)', async () => { + const federated = (await resolveSlugsOp.handler(federatedCtx(), { partial: 'topic' })) as string[]; + expect(federated).toContain('wiki/topic'); + expect(federated).not.toContain('private/topic'); + + // A remote scalar caller (no field, no grant) must no longer see foreign slugs. + const scalar = (await resolveSlugsOp.handler(ctxOf({ remote: true }), { partial: 'topic' })) as string[]; + expect(scalar).not.toContain('wiki/topic'); + expect(scalar).not.toContain('private/topic'); + expect(scalar).not.toContain('old/topic'); + }); +}); From 69bc37f745844794353d476891dfeaf2e0ace474 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:28:42 -0700 Subject: [PATCH 14/39] fix(auth): admin-gated rescope surface for DCR clients stuck on default scope (#3299) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(auth): admin-gated rescope surface for DCR clients stuck on default scope (#1914) DCR clients self-register with source_id='default' + federated_read=['default'] and the registration comment promised 'rescope via the CLI later' — but no rescope surface existed. Adds: - GBrainOAuthProvider.rescopeClient(clientId, { sourceId?, federatedRead? }): single-statement COALESCE update, canonical source-id validation (assertValidSourceId), FK-backed existence check on the write source, friendly errors for pre-v60/v61 schemas and unknown clients. Takes effect for already-issued tokens because verifyAccessToken re-reads oauth_clients. - gbrain auth rescope-client [--source S] [--federated-read a,b] (trusted local CLI). - POST /admin/api/rescope-client (requireAdmin), mirroring the existing register-client / revoke-client admin endpoints. Deliberately does NOT let clients self-widen scope (options a/b from the issue) — fail-closed trust invariant. Fixes #1914 Co-Authored-By: Claude Fable 5 * fix(auth): rescope-client admin endpoint returns 400 (not 500) for nonexistent write source The FK-translated 'Source "x" does not exist' error is a client error; map it to 400 like the sibling validation failures. ('No OAuth client found' is matched first, so the 404 path is unaffected.) Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Garry Tan Co-authored-by: Claude Fable 5 --- src/commands/auth.ts | 60 ++++++++++++++++++++++++++++++ src/commands/serve-http.ts | 32 ++++++++++++++++ src/core/oauth-provider.ts | 61 ++++++++++++++++++++++++++++++ test/oauth.test.ts | 76 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 229 insertions(+) diff --git a/src/commands/auth.ts b/src/commands/auth.ts index 6fefd6964..af759b576 100644 --- a/src/commands/auth.ts +++ b/src/commands/auth.ts @@ -515,6 +515,60 @@ async function registerClient(name: string, args: string[]) { } } +/** + * v0.42.x (#1914): rescope an existing OAuth client's write source and/or + * federated read scope. This is the operator surface the DCR registration + * comment promised ("rescope via the CLI later") — DCR clients land with + * source_id='default' / federated_read=['default'] and must not self-widen, + * so widening happens here (trusted local CLI) or via the requireAdmin + * /admin/api/rescope-client endpoint. + */ +async function rescopeClient(clientId: string, args: string[]) { + const usage = 'Usage: auth rescope-client [--source SOURCE] [--federated-read SRC1,SRC2,...]'; + if (!clientId) { + console.error(usage); + process.exit(1); + } + let sourceId: string | undefined; + let federatedRead: string[] | undefined; + for (let i = 0; i < args.length; i += 2) { + const flag = args[i]; + const value = args[i + 1]; + if (value === undefined || value.startsWith('--')) { + console.error(`Error: ${flag} requires a value`); + console.error(usage); + process.exit(1); + } + if (flag === '--source') sourceId = value; + else if (flag === '--federated-read') { + federatedRead = value.split(',').map(s => s.trim()).filter(Boolean); + } else { + console.error(`Error: Unknown flag: ${flag}`); + console.error(usage); + process.exit(1); + } + } + if (sourceId === undefined && federatedRead === undefined) { + console.error('Error: pass --source and/or --federated-read'); + console.error(usage); + process.exit(1); + } + try { + await withConfiguredSql(async (sql) => { + const { GBrainOAuthProvider } = await import('../core/oauth-provider.ts'); + const provider = new GBrainOAuthProvider({ sql }); + const result = await provider.rescopeClient(clientId, { sourceId, federatedRead }); + console.log(`OAuth client rescoped: "${result.clientName}" (${result.clientId})\n`); + console.log(` Write source: ${result.sourceId}`); + console.log(` Federated reads: ${result.federatedRead.join(', ') || ''}`); + console.log('\nTakes effect on the client\'s next request (existing tokens included).'); + }); + } catch (e: any) { + console.error('Error:', e.message); + process.exit(1); + } +} + /** * Entry point for the `gbrain auth` CLI subcommand. Also reused by the * direct-script path (see bottom of file) so `bun run src/commands/auth.ts` @@ -556,6 +610,7 @@ export async function runAuth(args: string[]): Promise { return; } case 'register-client': await registerClient(rest[0], rest.slice(1)); return; + case 'rescope-client': await rescopeClient(rest[0], rest.slice(1)); return; case 'revoke-client': await revokeClient(rest[0]); return; case 'test': { const tokenIdx = rest.indexOf('--token'); @@ -593,6 +648,11 @@ Usage: --bound-slug-prefixes Bind submit_agent writes to slug prefixes --bound-max-concurrent Bound submit_agent concurrency (default: 1) --budget-usd-per-day Bound submit_agent daily spend cap + gbrain auth rescope-client [options] Change an existing client's source scope (e.g. a DCR + client stuck on the 'default' source). Only the flags + you pass change; the other axis is left as-is. + --source New write source + --federated-read New read-scope source list gbrain auth revoke-client Hard-delete an OAuth 2.1 client (cascades to tokens + codes) gbrain auth test --token Smoke-test a remote MCP server `); diff --git a/src/commands/serve-http.ts b/src/commands/serve-http.ts index 994d9acfa..60c4f2ee2 100644 --- a/src/commands/serve-http.ts +++ b/src/commands/serve-http.ts @@ -1567,6 +1567,38 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption } }); + // v0.42.x (#1914): rescope an OAuth client's write source / federated read + // scope. Admin-gated on purpose — DCR clients must never self-widen their + // scope (fail-closed trust); only the operator rescopes, here or via + // `gbrain auth rescope-client`. Source ids are validated by the canonical + // validator inside rescopeClient. + app.post('/admin/api/rescope-client', requireAdmin, express.json(), async (req: Request, res: Response) => { + try { + const { clientId, sourceId, federatedRead } = req.body ?? {}; + if (!clientId || typeof clientId !== 'string') { + res.status(400).json({ error: 'clientId required' }); + return; + } + if (federatedRead !== undefined && + !(Array.isArray(federatedRead) && federatedRead.every((s: unknown) => typeof s === 'string'))) { + res.status(400).json({ error: 'federatedRead must be an array of source id strings' }); + return; + } + if (sourceId !== undefined && typeof sourceId !== 'string') { + res.status(400).json({ error: 'sourceId must be a string' }); + return; + } + const result = await oauthProvider.rescopeClient(clientId, { sourceId, federatedRead }); + res.json(result); + } catch (e) { + const message = e instanceof Error ? e.message : 'Rescope failed'; + const status = /No OAuth client found/.test(message) ? 404 + : /Invalid source_id|requires --source|cannot be empty|does not exist/.test(message) ? 400 + : 500; + res.status(status).json({ error: message }); + } + }); + // Revoke OAuth client app.post('/admin/api/revoke-client', requireAdmin, express.json(), async (req: Request, res: Response) => { try { diff --git a/src/core/oauth-provider.ts b/src/core/oauth-provider.ts index 46c2b16c7..17f383a7b 100644 --- a/src/core/oauth-provider.ts +++ b/src/core/oauth-provider.ts @@ -24,6 +24,7 @@ import type { OAuthRegisteredClientsStore } from '@modelcontextprotocol/sdk/serv import type { AuthInfo as SdkAuthInfo } from '@modelcontextprotocol/sdk/server/auth/types.js'; import { InvalidTokenError, InvalidClientMetadataError } from '@modelcontextprotocol/sdk/server/auth/errors.js'; import { hashToken, generateToken, isUndefinedColumnError } from './utils.ts'; +import { assertValidSourceId } from './source-id.ts'; import { hasScope, assertAllowedScopes, parseScopeString, InvalidScopeError } from './scope.ts'; import type { AuthInfo as CoreAuthInfo } from './operations.ts'; import { parseLegacyTokenScope } from './legacy-token-scope.ts'; @@ -1006,6 +1007,66 @@ export class GBrainOAuthProvider implements OAuthServerProvider { return { clientId, clientSecret }; } + /** + * v0.42.x (#1914): admin-gated rescope for an existing OAuth client. + * + * DCR clients self-register with source_id='default' + + * federated_read=['default'] and MUST NOT be able to widen their own + * scope (fail-closed trust). This is the trusted-operator surface that + * changes it afterward: `gbrain auth rescope-client` (local CLI) and + * POST /admin/api/rescope-client (requireAdmin) both route here. + * + * Omitted fields are left untouched (COALESCE). Takes effect on the + * client's NEXT request even for already-issued tokens, because + * verifyAccessToken re-reads oauth_clients on every verification. + */ + async rescopeClient( + clientId: string, + opts: { sourceId?: string; federatedRead?: string[] }, + ): Promise<{ clientId: string; clientName: string; sourceId: string; federatedRead: string[] }> { + const { sourceId, federatedRead } = opts; + if (sourceId === undefined && federatedRead === undefined) { + throw new Error('rescope-client requires --source and/or --federated-read'); + } + if (sourceId !== undefined) assertValidSourceId(sourceId); + if (federatedRead !== undefined) { + if (federatedRead.length === 0) { + throw new Error('--federated-read cannot be empty (pass at least one source id)'); + } + for (const s of federatedRead) assertValidSourceId(s); + } + let rows: Record[]; + try { + rows = await this.sql` + UPDATE oauth_clients + SET source_id = COALESCE(${sourceId ?? null}::text, source_id), + federated_read = COALESCE(${federatedRead ? pgArray(federatedRead) : null}::text[], federated_read) + WHERE client_id = ${clientId} + RETURNING client_id, client_name, source_id, federated_read + `; + } catch (err) { + if (isUndefinedColumnError(err, 'source_id') || isUndefinedColumnError(err, 'federated_read')) { + throw new Error('rescope-client requires an up-to-date OAuth schema; run `gbrain apply-migrations --yes` and retry.'); + } + // FK oauth_clients.source_id → sources(id): translate the raw 23503 + // into an actionable message. + if ((err as { code?: string })?.code === '23503') { + throw new Error(`Source "${sourceId}" does not exist. Create it first: gbrain sources add ${sourceId} ...`); + } + throw err; + } + if (rows.length === 0) { + throw new Error(`No OAuth client found with id "${clientId}"`); + } + const row = rows[0]; + return { + clientId: row.client_id as string, + clientName: (row.client_name as string | null) ?? '', + sourceId: (row.source_id as string | null) ?? 'default', + federatedRead: Array.isArray(row.federated_read) ? (row.federated_read as string[]) : [], + }; + } + // ------------------------------------------------------------------------- // Internal: Issue access + optional refresh tokens // ------------------------------------------------------------------------- diff --git a/test/oauth.test.ts b/test/oauth.test.ts index 43c484462..66eb7d3ee 100644 --- a/test/oauth.test.ts +++ b/test/oauth.test.ts @@ -166,6 +166,82 @@ describe('client registration', () => { }); }); +// --------------------------------------------------------------------------- +// rescopeClient (#1914) — admin-gated rescope of a DCR-defaulted client +// --------------------------------------------------------------------------- + +describe('rescopeClient', () => { + beforeAll(async () => { + // oauth_clients.source_id has FK → sources(id); create the targets. + for (const id of ['wiki', 'essays', 'alpha', 'gamma']) { + await sql`INSERT INTO sources (id, name) VALUES (${id}, ${id}) ON CONFLICT (id) DO NOTHING`; + } + }); + + test('DCR client stuck on default gets rescoped; existing tokens pick it up', async () => { + // Simulate the DCR path: self-registered client lands with + // source_id='default', federated_read=['default']. client_credentials + // over DCR needs the explicit --enable-dcr-insecure opt-in, so build a + // provider with that flag just for this registration. + const dcrProvider = new GBrainOAuthProvider({ sql, tokenTtl: 60, allowClientCredentialsDcr: true }); + const dcr = await dcrProvider.clientsStore.registerClient!({ + client_name: 'dcr-stuck-client', + redirect_uris: [], + grant_types: ['client_credentials'], + scope: 'read', + token_endpoint_auth_method: 'client_secret_post', + } as any); + const clientId = dcr.client_id; + const [before] = await sql`SELECT source_id, federated_read FROM oauth_clients WHERE client_id = ${clientId}`; + expect(before.source_id).toBe('default'); + expect(before.federated_read).toEqual(['default']); + + // Issue a token BEFORE the rescope — it must see the new scope after. + const tokens = await provider.exchangeClientCredentials(clientId, dcr.client_secret!, 'read'); + + const result = await provider.rescopeClient(clientId, { + sourceId: 'wiki', + federatedRead: ['wiki', 'essays'], + }); + expect(result.sourceId).toBe('wiki'); + expect(result.federatedRead).toEqual(['wiki', 'essays']); + + const authInfo = await provider.verifyAccessToken(tokens.access_token) as unknown as CoreAuthInfo; + expect(authInfo.sourceId).toBe('wiki'); + expect(authInfo.allowedSources).toEqual(['wiki', 'essays']); + }); + + test('partial rescope leaves the other axis untouched', async () => { + const { clientId } = await provider.registerClientManual( + 'partial-rescope', ['client_credentials'], 'read', [], 'alpha', ['alpha', 'beta'], + ); + const result = await provider.rescopeClient(clientId, { federatedRead: ['beta'] }); + expect(result.sourceId).toBe('alpha'); // untouched + expect(result.federatedRead).toEqual(['beta']); + + const result2 = await provider.rescopeClient(clientId, { sourceId: 'gamma' }); + expect(result2.sourceId).toBe('gamma'); + expect(result2.federatedRead).toEqual(['beta']); // untouched + }); + + test('rejects invalid source ids, empty federated list, no-op calls, unknown client', async () => { + const { clientId } = await provider.registerClientManual( + 'rescope-validation', ['client_credentials'], 'read', + ); + await expect(provider.rescopeClient(clientId, { sourceId: '../etc' })).rejects.toThrow('Invalid source_id'); + await expect(provider.rescopeClient(clientId, { federatedRead: ['ok', 'Not Valid!'] })).rejects.toThrow('Invalid source_id'); + await expect(provider.rescopeClient(clientId, { federatedRead: [] })).rejects.toThrow('cannot be empty'); + await expect(provider.rescopeClient(clientId, {})).rejects.toThrow('requires --source and/or --federated-read'); + await expect(provider.rescopeClient('gbrain_cl_nonexistent', { sourceId: 'wiki' })).rejects.toThrow('No OAuth client found'); + // FK: write source must exist in sources(id). + await expect(provider.rescopeClient(clientId, { sourceId: 'no-such-source' })).rejects.toThrow('does not exist'); + + // Validation failures must not have mutated the row. + const [row] = await sql`SELECT source_id FROM oauth_clients WHERE client_id = ${clientId}`; + expect(row.source_id).toBe('default'); + }); +}); + // --------------------------------------------------------------------------- // Client Credentials Exchange // --------------------------------------------------------------------------- From 40d9b83d5c2b8dd4b68ea3c19e02684a89d6c612 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:28:47 -0700 Subject: [PATCH 15/39] =?UTF-8?q?fix(cycle):=20wire=20drift=20detection=20?= =?UTF-8?q?into=20the=20dream=20cycle=20=E2=80=94=20report-only=20v1=20(#2?= =?UTF-8?q?653)=20(#3317)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dream.drift.enabled has gated an unwired scaffold since v0.28: runPhaseDrift had zero call sites, the resolved model + BudgetMeter were discarded (void modelId; void meter), and no operator-readable output existed. - Wire 'drift' as a CyclePhase (default OFF via dream.drift.enabled), ordered after the calibration trio and before embed so the report page gets embedded same-cycle. PHASE_SCOPE=global, cycle-lock coordinated, --once (--phase drift --once) bypasses the gate for one run. - Implement the LLM judge: soft-band candidates (weight 0.3-0.85, active, unresolved, fresh timeline evidence) are judged against their page's recent timeline entries via gateway chat; BudgetMeter-gated (dream.drift.budget, default $1), capped by dream.drift.max_per_cycle (default 20). Judge model resolves models.drift -> reasoning tier -> sonnet fallback (unchanged from scaffold). - Report-only v1: judged candidates land on a reports/drift- page. dream.drift.auto_update mutates NOTHING; the flag state is recorded in the report for operators. Co-authored-by: Garry Tan Co-authored-by: Claude Fable 5 --- src/commands/dream.ts | 4 +- src/core/cycle.ts | 56 ++++ src/core/cycle/drift.ts | 285 +++++++++++++++--- test/auto-think-phase.test.ts | 93 +++++- test/core/cycle.serial.test.ts | 7 +- .../dream-cycle-phase-order-pglite.test.ts | 1 + test/phase-scope-coverage.test.ts | 9 +- 7 files changed, 408 insertions(+), 47 deletions(-) diff --git a/src/commands/dream.ts b/src/commands/dream.ts index 71b598c6d..cc08b79f6 100644 --- a/src/commands/dream.ts +++ b/src/commands/dream.ts @@ -86,7 +86,7 @@ interface DreamArgs { * `--phase `; bare `--once` is a usage error (there'd be no single * phase to target). Applies only to phases with a config `.enabled` gate * (patterns, synthesize, conversation_facts_backfill, enrich_thin, - * skillopt) — a no-op for phases that always run when named directly. + * skillopt, drift) — a no-op for phases that always run when named directly. */ once: boolean; } @@ -367,7 +367,7 @@ Options: unlike toggling the flag on/off around the run, a crash mid-invocation can't leave it stuck. Applies to patterns, synthesize, conversation_facts_backfill, - enrich_thin, skillopt; no-op on phases with no such + enrich_thin, skillopt, drift; no-op on phases with no such gate. Requires an EXPLICIT --phase — a phase implied by --input or --drain does not count (bare --once, or --once with --input/--drain and no diff --git a/src/core/cycle.ts b/src/core/cycle.ts index f98e370e8..ce043006d 100644 --- a/src/core/cycle.ts +++ b/src/core/cycle.ts @@ -66,6 +66,10 @@ export type CyclePhase = // - calibration_profile: aggregates the resolved subset into 2-4 // narrative pattern statements + active bias tags. Voice-gated. | 'propose_takes' | 'grade_takes' | 'calibration_profile' + // #2653 — drift detection (default OFF; dream.drift.enabled). LLM-judges + // soft-band takes against recent timeline evidence; report-only in v1 + // (writes reports/drift-; auto_update mutates nothing). + | 'drift' | 'embed' | 'orphans' | 'purge' // v0.39 T12: schema-suggest passive trigger (D3 + D4 plan-eng-review). // Wraps runSuggest() — same library the CLI verb + EIIRP call. @@ -151,6 +155,10 @@ export const ALL_PHASES: CyclePhase[] = [ 'propose_takes', 'grade_takes', 'calibration_profile', + // #2653 — drift detection. Default OFF (dream.drift.enabled). Runs AFTER + // the calibration trio (fresh take resolutions) and BEFORE embed so the + // drift report page gets embedded same-cycle. Report-only in v1. + 'drift', // v0.41.11.0 — opt-in conversation-facts backfill. Default OFF; reads // cycle.conversation_facts_backfill.enabled gate inside the wrapper. // Ordered AFTER calibration_profile (matches the runCycle dispatch @@ -221,6 +229,9 @@ export const PHASE_SCOPE: Record = { propose_takes: 'source', grade_takes: 'global', calibration_profile: 'global', + // #2653 — drift walks takes brain-wide (same posture as grade_takes) and + // writes one brain-global report page. + drift: 'global', embed: 'global', orphans: 'global', purge: 'global', @@ -289,6 +300,8 @@ const NEEDS_LOCK_PHASES: ReadonlySet = new Set([ 'propose_takes', 'grade_takes', 'calibration_profile', + // #2653 — writes the reports/drift- page. + 'drift', // v0.41 T9 — extract_atoms writes atom-typed pages via put_page; // synthesize_concepts writes concept-typed pages + tier updates. Both // mutate DB state and need the lock. @@ -2151,6 +2164,49 @@ export async function runCycle( } } + // ── #2653: drift detection ────────────────────────────────── + // Default OFF (dream.drift.enabled). LLM-judges soft-band takes + // against recent timeline evidence; report-only in v1 — writes one + // reports/drift- page, mutates no takes regardless of + // dream.drift.auto_update. + if (phases.includes('drift')) { + checkAborted(opts.signal); + if (!engine) { + phaseResults.push({ + phase: 'drift', + status: 'skipped', + duration_ms: 0, + summary: 'no database connected', + details: { reason: 'no_database' }, + }); + } else { + progress.start('cycle.drift'); + const { runPhaseDrift } = await import('./cycle/drift.ts'); + const { result, duration_ms } = await timePhase(async (): Promise => { + const r = await runPhaseDrift(engine, { + dryRun, + brainDir: brainDir ?? undefined, + forceEnabled: opts.onceForPhase === 'drift', + }); + const status: PhaseStatus = + r.status === 'complete' ? 'ok' : + r.status === 'partial' ? 'warn' : + r.status === 'failed' ? 'fail' : 'skipped'; + return { + phase: 'drift', + status, + duration_ms: 0, + summary: r.detail, + details: { ...(r.totals ?? {}) }, + }; + }); + result.duration_ms = duration_ms; + phaseResults.push(result); + progress.finish(); + } + await safeYield(opts.yieldBetweenPhases); + } + // ── v0.41.11.0: conversation_facts_backfill ───────────────── // Opt-in (default OFF). Walks long-form conversation/meeting/slack/ // email pages, segments by 30-min gap, runs facts extractor with a diff --git a/src/core/cycle/drift.ts b/src/core/cycle/drift.ts index 4042e13d8..971a9e789 100644 --- a/src/core/cycle/drift.ts +++ b/src/core/cycle/drift.ts @@ -1,18 +1,25 @@ /** - * v0.28: drift dream phase. + * Drift dream phase (#2653 — wired v0.42.x; scaffold shipped v0.28). * - * Detects takes where the underlying evidence has shifted since the take - * was made. v0.28 ships the SCAFFOLD: the phase iterates active takes, - * runs a lightweight check against recent timeline entries on the same - * page, and writes a drift-report-.md if any takes look stale. + * Detects takes whose underlying evidence has shifted since the take was + * made. Two stages: + * 1. Cheap SQL heuristic (findDriftCandidates): soft-band takes + * (weight 0.3–0.85, active, unresolved) on pages with fresh + * timeline_entries evidence inside the lookback window. + * 2. LLM judge: each candidate's claim is compared against the recent + * timeline evidence; the judge returns {drifted, confidence, + * reasoning, suggested_weight}. BudgetMeter-gated. * - * The full LLM-driven drift detection (compare each take's claim to recent - * page evidence and propose a weight adjustment) is the v0.29 follow-up. - * v0.28 lays the phase orchestration so the contract is stable. + * Output is REPORT-ONLY (v1 conservative posture): judged candidates land + * on a `reports/drift-` page. `dream.drift.auto_update` mutates + * NOTHING in v1 — it is recorded in the report so operators can see the + * flag state, and a future wave may wire weight adjustment behind it. * * Default-disabled. Operator opts in: * gbrain config set dream.drift.enabled true * gbrain config set dream.drift.lookback_days 30 + * gbrain config set dream.drift.max_per_cycle 20 + * gbrain config set dream.drift.budget 1.0 */ import type { BrainEngine } from '../engine.ts'; @@ -25,6 +32,10 @@ export interface DriftPhaseOpts { dryRun: boolean; /** Override the audit ledger path (tests). */ auditPath?: string; + /** issue #2860 --once: bypass the dream.drift.enabled gate for this run only. */ + forceEnabled?: boolean; + /** Inject the judge model call (tests). Defaults to gateway chat. */ + judge?: DriftJudgeFn; } export interface DriftConfig { @@ -32,6 +43,7 @@ export interface DriftConfig { lookbackDays: number; budgetUsd: number; autoUpdate: boolean; + maxPerCycle: number; } async function loadDriftConfig(engine: BrainEngine): Promise { @@ -39,16 +51,19 @@ async function loadDriftConfig(engine: BrainEngine): Promise { const lookbackStr = await engine.getConfig('dream.drift.lookback_days'); const budgetStr = await engine.getConfig('dream.drift.budget'); const autoStr = await engine.getConfig('dream.drift.auto_update'); + const maxPerStr = await engine.getConfig('dream.drift.max_per_cycle'); return { enabled: enabledStr === 'true', lookbackDays: lookbackStr ? Math.max(1, parseInt(lookbackStr, 10) || 30) : 30, budgetUsd: budgetStr ? Math.max(0, parseFloat(budgetStr) || 1.0) : 1.0, autoUpdate: autoStr === 'true', + maxPerCycle: maxPerStr ? Math.max(1, parseInt(maxPerStr, 10) || 20) : 20, }; } -interface DriftCandidate { +export interface DriftCandidate { takeId: number; + pageId: number; pageSlug: string; rowNum: number; claim: string; @@ -57,24 +72,121 @@ interface DriftCandidate { recentEvidenceCount: number; } +/** Judge verdict: has the claim drifted relative to the evidence? */ +export interface DriftVerdict { + drifted: boolean; + confidence: number; + reasoning: string; + /** Judge's suggested new weight (advisory only — v1 never applies it). */ + suggested_weight?: number; +} + +/** Judge function signature — injected for tests. */ +export type DriftJudgeFn = (input: { + candidate: DriftCandidate; + evidence: string; + modelHint?: string; +}) => Promise; + +export const DRIFT_JUDGE_PROMPT = `You are auditing a knowledge-base "take" (a weighted claim) for drift: +has newer evidence shifted the ground under this claim since it was made? + +Output ONLY one JSON object with these fields: +- drifted (boolean) — true when the evidence meaningfully contradicts, + supersedes, or reframes the claim; false when it is + consistent or merely adjacent. +- confidence (number in [0,1]) — your confidence in the drifted verdict. +- reasoning (string, <=300 chars) — what in the evidence drove the verdict. +- suggested_weight (number in [0,1], optional) — where the take's weight should + move if drifted. Advisory only. + +If the evidence is sparse or unrelated to the claim, return drifted=false with +low confidence. + +TAKE: + Claim: {CLAIM} + Weight: {WEIGHT} + Page: {PAGE} + +RECENT EVIDENCE (timeline entries on the same page): +{EVIDENCE_BLOCK} +`; + +/** + * Parse the judge model's JSON output. Tolerant of fence wrapping and + * leading prose; returns null on unrecoverable parse failure. + */ +export function parseDriftOutput(raw: string): DriftVerdict | null { + if (!raw || raw.trim().length === 0) return null; + let text = raw.trim(); + const fenced = text.match(/^```(?:json)?\s*\n?([\s\S]*?)\n?```$/); + if (fenced) text = (fenced[1] ?? '').trim(); + const firstObj = text.indexOf('{'); + if (firstObj === -1) return null; + let parsed: unknown; + try { + parsed = JSON.parse(text.slice(firstObj)); + } catch { + return null; + } + if (typeof parsed !== 'object' || parsed === null) return null; + const r = parsed as Record; + if (typeof r.drifted !== 'boolean') return null; + const confRaw = typeof r.confidence === 'number' ? r.confidence : Number.parseFloat(String(r.confidence ?? '')); + if (!Number.isFinite(confRaw)) return null; + const verdict: DriftVerdict = { + drifted: r.drifted, + confidence: Math.max(0, Math.min(1, confRaw)), + reasoning: typeof r.reasoning === 'string' ? r.reasoning.slice(0, 300) : '', + }; + const sw = typeof r.suggested_weight === 'number' ? r.suggested_weight : NaN; + if (Number.isFinite(sw)) verdict.suggested_weight = Math.max(0, Math.min(1, sw)); + return verdict; +} + +/** Production judge — calls gateway.chat with the DRIFT_JUDGE_PROMPT. */ +export async function defaultDriftJudge(input: { + candidate: DriftCandidate; + evidence: string; + modelHint?: string; +}): Promise { + const { chat } = await import('../ai/gateway.ts'); + const prompt = DRIFT_JUDGE_PROMPT + .replace('{CLAIM}', input.candidate.claim) + .replace('{WEIGHT}', String(input.candidate.weight)) + .replace('{PAGE}', input.candidate.pageSlug) + .replace('{EVIDENCE_BLOCK}', input.evidence); + const result = await chat({ + messages: [{ role: 'user', content: prompt }], + ...(input.modelHint ? { model: input.modelHint } : {}), + maxTokens: 400, + }); + const parsed = parseDriftOutput(result.text); + if (!parsed) { + // Failed parse — conservative no-drift at zero confidence so the row + // still surfaces in the report instead of disappearing silently. + return { drifted: false, confidence: 0, reasoning: 'judge_output_parse_failed' }; + } + return parsed; +} + /** * Cheap pre-LLM heuristic: takes that have substantial recent timeline - * evidence on the same page MAY have drifted. Surface them; the v0.29 - * LLM judge will decide if the weight should move. + * evidence on the same page MAY have drifted. Surface them; the LLM judge + * decides. */ async function findDriftCandidates( engine: BrainEngine, lookbackDays: number, ): Promise { - const cutoffMs = Date.now() - lookbackDays * 86_400_000; - const cutoffIso = new Date(cutoffMs).toISOString().slice(0, 10); + const cutoffIso = lookbackCutoffIso(lookbackDays); // Only consider takes with weight in the "soft" middle band (0.3..0.85) // — facts (1.0) don't drift, very-low hunches (<0.3) aren't actionable yet. const rows = await engine.executeRaw<{ - take_id: number; page_slug: string; row_num: number; + take_id: number; page_id: number; page_slug: string; row_num: number; claim: string; weight: number; recent_evidence: number; }>(` - SELECT t.id AS take_id, p.slug AS page_slug, t.row_num, + SELECT t.id AS take_id, p.id AS page_id, p.slug AS page_slug, t.row_num, t.claim, t.weight, (SELECT count(*)::int FROM timeline_entries te WHERE te.page_id = p.id @@ -92,6 +204,7 @@ async function findDriftCandidates( .filter(r => Number(r.recent_evidence) >= 1) .map(r => ({ takeId: Number(r.take_id), + pageId: Number(r.page_id), pageSlug: String(r.page_slug), rowNum: Number(r.row_num), claim: String(r.claim), @@ -100,6 +213,58 @@ async function findDriftCandidates( })); } +function lookbackCutoffIso(lookbackDays: number): string { + return new Date(Date.now() - lookbackDays * 86_400_000).toISOString().slice(0, 10); +} + +/** Format the candidate page's recent timeline entries as judge evidence. */ +async function loadEvidence( + engine: BrainEngine, + pageId: number, + cutoffIso: string, +): Promise { + const rows = await engine.executeRaw<{ date: string; source: string; summary: string }>( + `SELECT date::text AS date, source, summary FROM timeline_entries + WHERE page_id = $1 AND date >= $2::date + ORDER BY date DESC LIMIT 12`, + [pageId, cutoffIso], + ); + if (rows.length === 0) return '(no timeline entries in window)'; + return rows.map(r => `- ${r.date} [${r.source}] ${r.summary}`).join('\n'); +} + +interface JudgedCandidate { + candidate: DriftCandidate; + verdict: DriftVerdict; +} + +function buildReportBody( + judged: JudgedCandidate[], + cfg: DriftConfig, + modelId: string, +): string { + const drifted = judged.filter(j => j.verdict.drifted); + const lines: string[] = [ + `Drift check over active soft-band takes (weight 0.3–0.85) with timeline`, + `evidence from the last ${cfg.lookbackDays} day(s). Judge model: ${modelId}.`, + ``, + `**Report-only:** no takes were modified. \`dream.drift.auto_update\` is`, + `${cfg.autoUpdate ? 'set but ignored in v1 (report-only posture)' : 'off'}; review and adjust weights manually.`, + ``, + `${drifted.length} of ${judged.length} judged take(s) look drifted.`, + ``, + ]; + for (const { candidate: c, verdict: v } of judged) { + lines.push(`## ${v.drifted ? 'DRIFTED' : 'stable'} — ${c.pageSlug} (take #${c.rowNum})`); + lines.push(`- Claim: ${c.claim}`); + lines.push(`- Weight: ${c.weight}${v.suggested_weight !== undefined ? ` → suggested ${v.suggested_weight}` : ''}`); + lines.push(`- Confidence: ${v.confidence.toFixed(2)}`); + if (v.reasoning) lines.push(`- Reasoning: ${v.reasoning}`); + lines.push(''); + } + return lines.join('\n'); +} + function skipped(_reason: string, detail: string): DreamPhaseResult { return { name: 'drift', status: 'skipped', detail, duration_ms: 0 }; } @@ -110,7 +275,7 @@ export async function runPhaseDrift( ): Promise { const start = Date.now(); const config = await loadDriftConfig(engine); - if (!config.enabled) { + if (!config.enabled && !opts.forceEnabled) { return skipped('not_configured', 'dream.drift.enabled is false'); } @@ -125,9 +290,16 @@ export async function runPhaseDrift( }; } - // Resolve model for the (future v0.29) LLM judge. For v0.28 we just - // surface the candidates — the meter call is a no-op when we don't actually - // submit, but resolveModel sets the right pricing key when v0.29 ships. + if (opts.dryRun) { + return { + name: 'drift', + status: 'skipped', + detail: `dry-run: ${Math.min(candidates.length, config.maxPerCycle)} of ${candidates.length} candidates would be evaluated`, + totals: { candidates: candidates.length }, + duration_ms: Date.now() - start, + }; + } + const modelId = await resolveModel(engine, { configKey: 'models.drift', deprecatedConfigKey: 'dream.drift.model', @@ -139,27 +311,70 @@ export async function runPhaseDrift( phase: 'drift', auditPath: opts.auditPath, }); + const judge = opts.judge ?? defaultDriftJudge; + const cutoffIso = lookbackCutoffIso(config.lookbackDays); - // v0.28 scaffold: write a candidate report. v0.29 wires LLM-driven weight - // adjustment through autoUpdate. modelId + meter are wired now so the - // ledger captures the gate state even when we don't submit. - void modelId; void meter; - - if (opts.dryRun) { - return { - name: 'drift', - status: 'skipped', - detail: `dry-run: ${candidates.length} candidates would be evaluated`, - totals: { candidates: candidates.length }, - duration_ms: Date.now() - start, - }; + const judged: JudgedCandidate[] = []; + let budgetExhausted = false; + let failed = 0; + for (const candidate of candidates.slice(0, config.maxPerCycle)) { + const check = meter.check({ + modelId, + estimatedInputTokens: 1500, + maxOutputTokens: 400, + label: `drift:${candidate.pageSlug}#${candidate.rowNum}`, + }); + if (!check.allowed) { + budgetExhausted = true; + break; + } + const evidence = await loadEvidence(engine, candidate.pageId, cutoffIso); + try { + const verdict = await judge({ candidate, evidence, modelHint: modelId }); + judged.push({ candidate, verdict }); + } catch (e) { + failed += 1; + process.stderr.write(`[drift] judge failed on take ${candidate.takeId}: ${(e as Error).message}\n`); + } } + const driftedCount = judged.filter(j => j.verdict.drifted).length; + let reportSlug: string | undefined; + if (judged.length > 0) { + const date = new Date().toISOString().slice(0, 10); + reportSlug = `reports/drift-${date}`; + // Report-only v1: the report page is the ONLY write this phase makes. + // Lands in the default source (brain-global artifact, same-day re-runs + // upsert the same slug). + await engine.putPage(reportSlug, { + type: 'report', + title: `Drift report ${date}`, + compiled_truth: buildReportBody(judged, config, modelId), + }); + } + + const detail = + `judged ${judged.length}/${candidates.length} candidates: ${driftedCount} drifted` + + (reportSlug ? ` → ${reportSlug}` : '') + + (budgetExhausted ? ' (budget exhausted)' : '') + + (failed > 0 ? ` (${failed} judge failure(s))` : '') + + `. Cumulative cost: $${meter.totalSpent.toFixed(4)} / $${config.budgetUsd.toFixed(2)}` + + `. Report-only: auto_update=${config.autoUpdate} mutates nothing in v1.`; + return { name: 'drift', - status: 'complete', - detail: `surfaced ${candidates.length} drift candidates (LLM judge: v0.29 follow-up). autoUpdate=${config.autoUpdate}`, - totals: { candidates: candidates.length }, + status: judged.length > 0 + ? (budgetExhausted || failed > 0 ? 'partial' : 'complete') + // Zero judged: budget capped before any judge ran → partial (capped, + // not broken); otherwise every judge call failed → failed. + : (budgetExhausted ? 'partial' : 'failed'), + detail, + totals: { + candidates: candidates.length, + judged: judged.length, + drifted: driftedCount, + failed, + }, duration_ms: Date.now() - start, }; } diff --git a/test/auto-think-phase.test.ts b/test/auto-think-phase.test.ts index f46462c2e..4d2d05f62 100644 --- a/test/auto-think-phase.test.ts +++ b/test/auto-think-phase.test.ts @@ -4,7 +4,7 @@ import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { PGLiteEngine } from '../src/core/pglite-engine.ts'; import { runPhaseAutoThink } from '../src/core/cycle/auto-think.ts'; -import { runPhaseDrift, __testing as driftTesting } from '../src/core/cycle/drift.ts'; +import { runPhaseDrift, parseDriftOutput, __testing as driftTesting } from '../src/core/cycle/drift.ts'; import { _resetBudgetMeterWarningsForTest } from '../src/core/cycle/budget-meter.ts'; import type { ThinkLLMClient } from '../src/core/think/index.ts'; @@ -153,6 +153,18 @@ describe('runPhaseAutoThink', () => { }); }); +describe('parseDriftOutput', () => { + test('parses fenced JSON with clamped fields', () => { + const v = parseDriftOutput('```json\n{"drifted": true, "confidence": 1.7, "reasoning": "r", "suggested_weight": -0.2}\n```'); + expect(v).toEqual({ drifted: true, confidence: 1, reasoning: 'r', suggested_weight: 0 }); + }); + + test('returns null on garbage / missing drifted', () => { + expect(parseDriftOutput('not json at all')).toBeNull(); + expect(parseDriftOutput('{"confidence": 0.5}')).toBeNull(); + }); +}); + describe('runPhaseDrift', () => { test('skipped when not enabled', async () => { const r = await runPhaseDrift(engine, { dryRun: false, auditPath: join(tmpDir, 'd0.jsonl') }); @@ -166,16 +178,89 @@ describe('runPhaseDrift', () => { expect(cands.every(c => c.weight >= 0.3 && c.weight <= 0.85)).toBe(true); }); - test('runs and surfaces candidates when enabled', async () => { + test('judges candidates and writes a report-only drift report page (#2653)', async () => { await engine.setConfig('dream.drift.enabled', 'true'); await engine.setConfig('dream.drift.lookback_days', '30'); await engine.setConfig('dream.drift.budget', '1.0'); - const r = await runPhaseDrift(engine, { dryRun: false, auditPath: join(tmpDir, 'd1.jsonl') }); + const judgedRows: number[] = []; + const r = await runPhaseDrift(engine, { + dryRun: false, + auditPath: join(tmpDir, 'd1.jsonl'), + judge: async ({ candidate, evidence }) => { + judgedRows.push(candidate.rowNum); + expect(evidence).toContain('Funding round closed'); // real timeline evidence reached the judge + return candidate.rowNum === 2 + ? { drifted: true, confidence: 0.9, reasoning: 'evidence shifted', suggested_weight: 0.3 } + : { drifted: false, confidence: 0.7, reasoning: 'consistent' }; + }, + }); expect(r.status).toBe('complete'); - expect((r.totals as { candidates?: number }).candidates).toBeGreaterThanOrEqual(0); + const totals = r.totals as { candidates: number; judged: number; drifted: number }; + expect(totals.judged).toBeGreaterThanOrEqual(2); + expect(totals.drifted).toBe(1); + expect(judgedRows).toContain(2); + + // Report page landed. + const date = new Date().toISOString().slice(0, 10); + const page = await engine.getPage(`reports/drift-${date}`); + expect(page).not.toBeNull(); + expect(page!.compiled_truth).toContain('DRIFTED'); + expect(page!.compiled_truth).toContain('Strong technical founder'); + + // Report-only v1: no take was mutated even though the judge suggested a weight. + const rows = await engine.executeRaw<{ weight: number; resolved_at: string | null }>( + 'SELECT weight, resolved_at FROM takes WHERE page_id = $1 AND row_num = 2', + [alicePageId], + ); + expect(Number(rows[0]!.weight)).toBe(0.6); + expect(rows[0]!.resolved_at).toBeNull(); await engine.setConfig('dream.drift.enabled', 'false'); }); + test('auto_update mutates nothing in v1', async () => { + await engine.setConfig('dream.drift.enabled', 'true'); + await engine.setConfig('dream.drift.auto_update', 'true'); + const r = await runPhaseDrift(engine, { + dryRun: false, + auditPath: join(tmpDir, 'd-auto.jsonl'), + judge: async () => ({ drifted: true, confidence: 0.99, reasoning: 'x', suggested_weight: 0.1 }), + }); + expect(r.status).toBe('complete'); + const rows = await engine.executeRaw<{ weight: number }>( + 'SELECT weight FROM takes WHERE page_id = $1 AND row_num = 3', + [alicePageId], + ); + expect(Number(rows[0]!.weight)).toBe(0.5); // untouched + await engine.setConfig('dream.drift.auto_update', 'false'); + await engine.setConfig('dream.drift.enabled', 'false'); + }); + + test('budget exhaustion stops judging (partial, no judge calls)', async () => { + await engine.setConfig('dream.drift.enabled', 'true'); + await engine.setConfig('dream.drift.budget', '0.0000001'); + let judgeCalls = 0; + const r = await runPhaseDrift(engine, { + dryRun: false, + auditPath: join(tmpDir, 'd-budget.jsonl'), + judge: async () => { judgeCalls += 1; return { drifted: false, confidence: 0.5, reasoning: '' }; }, + }); + expect(r.status).toBe('partial'); + expect(judgeCalls).toBe(0); + await engine.setConfig('dream.drift.budget', '1.0'); + await engine.setConfig('dream.drift.enabled', 'false'); + }); + + test('forceEnabled (--once) bypasses the dream.drift.enabled gate', async () => { + await engine.setConfig('dream.drift.enabled', 'false'); + const r = await runPhaseDrift(engine, { + dryRun: false, + auditPath: join(tmpDir, 'd-once.jsonl'), + forceEnabled: true, + judge: async () => ({ drifted: false, confidence: 0.5, reasoning: 'ok' }), + }); + expect(r.status).toBe('complete'); + }); + test('dry-run returns skipped with candidate count', async () => { await engine.setConfig('dream.drift.enabled', 'true'); const r = await runPhaseDrift(engine, { dryRun: true, auditPath: join(tmpDir, 'd2.jsonl') }); diff --git a/test/core/cycle.serial.test.ts b/test/core/cycle.serial.test.ts index 247124d58..8676bbd67 100644 --- a/test/core/cycle.serial.test.ts +++ b/test/core/cycle.serial.test.ts @@ -394,7 +394,9 @@ describe('runCycle — yieldBetweenPhases hook', () => { // v0.41.11.0: 20 phases (added `conversation_facts_backfill` between consolidate and propose_takes). // v0.41.39 (#1700) + v0.42.0.0: 22 phases (added `enrich_thin` AND `skillopt` // between conversation_facts_backfill and embed — both landed in this merge). - expect(hookCalls).toBe(22); + // #2653: 23 phases (added `drift` between calibration_profile and + // conversation_facts_backfill). + expect(hookCalls).toBe(23); }); test('hook exceptions do not abort the cycle', async () => { @@ -409,7 +411,8 @@ describe('runCycle — yieldBetweenPhases hook', () => { // v0.39.0.0: 17 phases (T12 schema-suggest phase between orphans and purge). // v0.41.11.0: 20 phases (+extract_atoms, +synthesize_concepts, +conversation_facts_backfill). // v0.41.39 (#1700) + v0.42.0.0: 22 phases (+enrich_thin, +skillopt). - expect(report.phases.length).toBe(22); + // #2653: 23 phases (+drift). + expect(report.phases.length).toBe(23); }); }); diff --git a/test/e2e/dream-cycle-phase-order-pglite.test.ts b/test/e2e/dream-cycle-phase-order-pglite.test.ts index 53d8add53..c9356d752 100644 --- a/test/e2e/dream-cycle-phase-order-pglite.test.ts +++ b/test/e2e/dream-cycle-phase-order-pglite.test.ts @@ -127,6 +127,7 @@ const EXPECTED_PHASES: CyclePhase[] = [ 'propose_takes', // v0.36.1.0 — hindsight calibration wave 'grade_takes', // v0.36.1.0 'calibration_profile', // v0.36.1.0 + 'drift', // #2653 — drift detection (default OFF, report-only) 'conversation_facts_backfill', // v0.41.11.0 — opt-in conversation backfill 'enrich_thin', // v0.41.39 (#1700) — brain-internal stub enrichment (default OFF) 'skillopt', // v0.42.0.0 — self-evolving skills (default OFF) diff --git a/test/phase-scope-coverage.test.ts b/test/phase-scope-coverage.test.ts index ccd58b326..957caf47a 100644 --- a/test/phase-scope-coverage.test.ts +++ b/test/phase-scope-coverage.test.ts @@ -41,15 +41,16 @@ describe('PHASE_SCOPE coverage', () => { expect(invalid).toEqual([]); }); - test('all 22 phases covered (regression on accidental omission)', () => { + test('all 23 phases covered (regression on accidental omission)', () => { // Pin the count so a future PR that adds a phase to ALL_PHASES // without updating PHASE_SCOPE notices here too. The v0.39.1.0 // master merge brought in the 17th phase (`schema-suggest`); v0.41 // adds 'extract_atoms' + 'synthesize_concepts' (T9 lens packs) + // 'conversation_facts_backfill' (v0.41.11.0) for 20; v0.41.39 (#1700) - // adds 'enrich_thin' and v0.42.0.0 adds 'skillopt' for a total of 22. - expect(ALL_PHASES.length).toBe(22); - expect(Object.keys(PHASE_SCOPE).length).toBe(22); + // adds 'enrich_thin' and v0.42.0.0 adds 'skillopt' for a total of 22; + // #2653 adds 'drift' for 23. + expect(ALL_PHASES.length).toBe(23); + expect(Object.keys(PHASE_SCOPE).length).toBe(23); }); test('embed remains global (the headline brain-wide phase)', () => { From aae1a5107e5be43c0aefaded47b2a57031e4a6f8 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:28:51 -0700 Subject: [PATCH 16/39] =?UTF-8?q?fix(doctor):=20raw-source=20persistence?= =?UTF-8?q?=20guarantee=20for=20synthesized=20pages=20=E2=80=94=20warn-onl?= =?UTF-8?q?y=20v1=20(#3300)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(doctor): raw-source persistence guarantee — warn-only v1 (#1978) Every synthesized/derived page (dream_generated:true or type:synthesis) must carry a raw trace or an explicit exemption. v1 is warn-only: - New doctor check `raw_provenance` (brain category) flags synthesized pages with none of: raw_trace/raw_source/source_uri/raw_trace_exempt frontmatter, an attached raw_data row, or synthesis_evidence rows. - Dream synthesize now stamps `raw_source: ` into each written page's frontmatter via the existing #2569 provenance stamp. - Dream-cycle summary index pages and extract receipts carry an explicit `raw_trace_exempt: true` + reason (no source document of their own). No write path is blocked; fail-closed enforcement is the v2 escalation. Co-Authored-By: Claude Fable 5 * fix(doctor): exclude soft-deleted pages from raw_provenance check Sibling frontmatter checks (quarantined_pages, flagged_pages) filter deleted_at IS NULL; without it a deleted synthesized page keeps warning (and its slug keeps being named) through the 72h recovery window with no way to clear the warn. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Garry Tan Co-authored-by: Claude Fable 5 --- src/commands/doctor.ts | 61 ++++++++++ src/core/cycle/synthesize.ts | 45 +++++-- src/core/doctor-categories.ts | 1 + src/core/extract/receipt-writer.ts | 5 + test/cycle-synthesize-slug-collection.test.ts | 43 +++++++ test/doctor-raw-provenance.test.ts | 110 ++++++++++++++++++ test/extract/receipt-writer.test.ts | 5 + 7 files changed, 261 insertions(+), 9 deletions(-) create mode 100644 test/doctor-raw-provenance.test.ts diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index bc84b4738..5b917f9fa 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -529,6 +529,61 @@ export async function childTableOrphansCheck(engine: BrainEngine): Promise { + const where = ` + p.deleted_at IS NULL + AND (COALESCE(p.frontmatter->>'dream_generated', '') = 'true' OR p.type = 'synthesis') + AND NOT (COALESCE(p.frontmatter, '{}'::jsonb) ?| ARRAY['raw_trace', 'raw_source', 'source_uri', 'raw_trace_exempt']) + AND NOT EXISTS (SELECT 1 FROM raw_data rd WHERE rd.page_id = p.id) + AND NOT EXISTS (SELECT 1 FROM synthesis_evidence se WHERE se.synthesis_page_id = p.id)`; + try { + const rows = await engine.executeRaw<{ n: string | number }>( + `SELECT COUNT(*)::int AS n FROM pages p WHERE ${where}`, + ); + const n = Number(rows[0]?.n ?? 0); + if (n === 0) { + return { + name: 'raw_provenance', + status: 'ok', + message: 'All synthesized pages carry a raw trace or explicit exemption', + }; + } + const sample = await engine.executeRaw<{ slug: string }>( + `SELECT p.slug FROM pages p WHERE ${where} ORDER BY p.slug LIMIT 5`, + ); + const slugs = sample.map(r => r.slug).join(', '); + return { + name: 'raw_provenance', + status: 'warn', + message: + `${n} synthesized page(s) lack a raw trace (no raw_trace/raw_source/source_uri frontmatter, ` + + `raw_data row, or synthesis evidence) and carry no raw_trace_exempt marker. e.g. ${slugs}. ` + + `Fix: stamp raw_source (path/URI of the source material) or raw_trace_exempt: true + ` + + `raw_trace_exempt_reason in frontmatter. Warn-only (#1978).`, + }; + } catch { + return { name: 'raw_provenance', status: 'warn', message: 'Could not check raw provenance (older schema?)' }; + } +} + export async function doctorReportRemote(engine: BrainEngine): Promise { const checks: Check[] = []; @@ -6290,6 +6345,12 @@ export async function buildChecks( progress.heartbeat('child_table_orphans'); checks.push(await childTableOrphansCheck(engine)); + // 10d. Raw-source persistence guarantee (#1978, warn-only v1). + // Every synthesized/derived page must carry a raw trace or an explicit + // exemption. Warn-only in v1 — surfaces violations, blocks nothing. + progress.heartbeat('raw_provenance'); + checks.push(await rawProvenanceCheck(engine)); + // v0.33: whoknows_health — fixture presence + row count. The eval // gate itself runs via `gbrain eval whoknows`; this check is the // "did you do the assignment?" signal. diff --git a/src/core/cycle/synthesize.ts b/src/core/cycle/synthesize.ts index dea310c63..bf680dbc7 100644 --- a/src/core/cycle/synthesize.ts +++ b/src/core/cycle/synthesize.ts @@ -554,6 +554,8 @@ export async function runPhaseSynthesize( const childIds: number[] = []; /** Map child job_id → chunk metadata for D6 orchestrator-side slug rewrite. */ const chunkInfo = new Map(); + /** #1978: map child job_id → source transcript path so written pages get a raw_source stamp. */ + const jobRawSource = new Map(); /** Skip reasons for the cycle report (D5 cap hits, D8 legacy-key skips). */ const skipReports: Array<{ filePath: string; reason: string }> = []; @@ -638,6 +640,7 @@ export async function runPhaseSynthesize( { allowProtectedSubmit: true }, ); childIds.push(child.id); + jobRawSource.set(child.id, t.filePath); if (isChunked) { chunkInfo.set(child.id, { idx: i, hash6 }); } @@ -682,7 +685,7 @@ export async function runPhaseSynthesize( // (source, slug) row. #1586: refs are stamped with the cycle's resolved // source (children write there via SubagentHandlerData.source_id). const cycleSourceId = opts.sourceId ?? 'default'; - const writtenRefs = await collectChildPutPageSlugs(engine, childIds, chunkInfo, cycleSourceId); + const writtenRefs = await collectChildPutPageSlugs(engine, childIds, chunkInfo, cycleSourceId, jobRawSource); const summaryDate = opts.date ?? today(); @@ -1234,7 +1237,8 @@ async function collectChildPutPageSlugs( childIds: number[], chunkInfo: Map, sourceId = 'default', -): Promise> { + jobRawSource?: Map, +): Promise> { if (childIds.length === 0) return []; // Raw fetch — NO SELECT DISTINCT. Preserves per-child slug duplicates so // the orchestrator sees what each child wrote. COALESCE handles both @@ -1256,13 +1260,21 @@ async function collectChildPutPageSlugs( AND status = 'complete'`, [childIds], ); - const rewritten = new Set(); + // #1978: slug → source transcript path (first writer wins) so the + // provenance stamp can record WHERE the synthesized content came from. + const rewritten = new Map(); for (const r of rows) { if (typeof r.slug !== 'string' || r.slug.length === 0) continue; const ci = chunkInfo.get(r.job_id); - rewritten.add(ci ? rewriteChunkedSlug(r.slug, ci.hash6, ci.idx) : r.slug); + const slug = ci ? rewriteChunkedSlug(r.slug, ci.hash6, ci.idx) : r.slug; + if (!rewritten.has(slug) || rewritten.get(slug) === undefined) { + rewritten.set(slug, jobRawSource?.get(r.job_id)); + } } - return Array.from(rewritten).sort().map(slug => ({ slug, source_id: sourceId })); + return Array.from(rewritten.keys()).sort().map(slug => { + const raw_source = rewritten.get(slug); + return { slug, source_id: sourceId, ...(raw_source ? { raw_source } : {}) }; + }); } /** @@ -1308,12 +1320,12 @@ async function hasLegacySingleChunkCompletion( */ async function stampDreamProvenance( engine: BrainEngine, - refs: Array<{ slug: string; source_id: string }>, + refs: Array<{ slug: string; source_id: string; raw_source?: string }>, cycleDate: string, ): Promise { if (refs.length === 0) return; const { executeRawJsonb } = await import('../sql-query.ts'); - for (const { slug, source_id } of refs) { + for (const { slug, source_id, raw_source } of refs) { try { await executeRawJsonb( engine, @@ -1321,7 +1333,14 @@ async function stampDreamProvenance( SET frontmatter = COALESCE(frontmatter, '{}'::jsonb) || $3::jsonb WHERE slug = $1 AND source_id = $2`, [slug, source_id], - [{ dream_generated: true, dream_cycle_date: cycleDate }], + // #1978 raw-source persistence: record the transcript path the + // synthesis was derived from, so `gbrain doctor` (raw_provenance + // check) can verify every generated page carries a raw trace. + [{ + dream_generated: true, + dream_cycle_date: cycleDate, + ...(raw_source ? { raw_source } : {}), + }], ); } catch (e) { const msg = e instanceof Error ? e.message : String(e); @@ -1423,7 +1442,15 @@ async function writeSummaryPage( // parseMarkdown below round-trips it into the DB-stored frontmatter, so the // marker survives any later reverse-render of the summary page. const fullMarkdown = serializeMarkdown( - { dream_generated: true, dream_cycle_date: summaryDate } as Record, + { + dream_generated: true, + dream_cycle_date: summaryDate, + // #1978: deterministic index page — no source document of its own; + // raw traces live on the listed pages. Explicit exemption keeps the + // doctor raw_provenance check quiet. + raw_trace_exempt: true, + raw_trace_exempt_reason: 'deterministic dream-cycle index; raw traces live on listed pages', + } as Record, body, '', { type: 'note' as string, title: `Dream cycle ${summaryDate}`, tags: ['dream-cycle'] }, diff --git a/src/core/doctor-categories.ts b/src/core/doctor-categories.ts index a59a7ea9d..c4d99ffa4 100644 --- a/src/core/doctor-categories.ts +++ b/src/core/doctor-categories.ts @@ -99,6 +99,7 @@ export const BRAIN_CHECK_NAMES: ReadonlySet = new Set([ 'orphan_ratio', 'oversized_pages', 'quarantined_pages', + 'raw_provenance', 'flagged_pages', 'salience_health', 'scraper_junk_pages', diff --git a/src/core/extract/receipt-writer.ts b/src/core/extract/receipt-writer.ts index 1334695ea..be4961ae7 100644 --- a/src/core/extract/receipt-writer.ts +++ b/src/core/extract/receipt-writer.ts @@ -157,6 +157,11 @@ function buildReceiptFrontmatter(input: ExtractReceiptInput): Record = { type: 'extract_receipt', dream_generated: true, + // #1978: receipts record an operation, not a source document — the + // run_id/round fields ARE the provenance. Explicit exemption keeps the + // doctor raw_provenance check quiet. + raw_trace_exempt: true, + raw_trace_exempt_reason: 'operation receipt; provenance is run_id + round', kind: input.kind, source_id: input.source_id, run_id: input.run_id, diff --git a/test/cycle-synthesize-slug-collection.test.ts b/test/cycle-synthesize-slug-collection.test.ts index 1ccbaa27e..d83ad4b3d 100644 --- a/test/cycle-synthesize-slug-collection.test.ts +++ b/test/cycle-synthesize-slug-collection.test.ts @@ -117,6 +117,22 @@ describe('C6: collectChildPutPageSlugs survives double-encoded jsonb (#745)', () expect(refs.length).toBeGreaterThan(0); for (const r of refs) expect(r.source_id).toBe('default'); }); + + // #1978: refs carry the source transcript path when the orchestrator + // supplies a job_id → path map, so stampDreamProvenance can persist it. + test('stamps refs with raw_source from the jobRawSource map (#1978)', async () => { + const jobRawSource = new Map([[1001, '/transcripts/2026-07-01-standup.md']]); + const refs = await collectChildPutPageSlugs(engine as any, [1001], new Map(), 'default', jobRawSource); + const ref = refs.find((r: { slug: string }) => r.slug === 'wiki/agents/test/normal-shape'); + expect(ref?.raw_source).toBe('/transcripts/2026-07-01-standup.md'); + }); + + test('omits raw_source when no map entry exists for the job (#1978)', async () => { + const refs = await collectChildPutPageSlugs(engine as any, [1001], new Map(), 'default', new Map()); + const ref = refs.find((r: { slug: string }) => r.slug === 'wiki/agents/test/normal-shape'); + expect(ref).toBeDefined(); + expect('raw_source' in (ref as object)).toBe(false); + }); }); describe('#2569: stampDreamProvenance persists the marker into DB frontmatter', () => { @@ -151,4 +167,31 @@ describe('#2569: stampDreamProvenance persists the marker into DB frontmatter', await stampDreamProvenance(engine as any, refs, '2026-07-17'); // no throw await stampDreamProvenance(engine as any, refs, '2026-07-17'); // idempotent }); + + // #1978: raw-source persistence — the stamp carries the transcript path + // the synthesis was derived from, when the ref supplies one. + test('persists raw_source into pages.frontmatter when the ref carries it (#1978)', async () => { + await engine.putPage('wiki/originals/ideas/2026-07-17-raw-src-def456', { + type: 'note', + title: 'Raw source stamp', + compiled_truth: 'body', + timeline: '', + frontmatter: {}, + }); + await stampDreamProvenance( + engine as any, + [{ + slug: 'wiki/originals/ideas/2026-07-17-raw-src-def456', + source_id: 'default', + raw_source: '/transcripts/2026-07-17-standup.md', + }], + '2026-07-17', + ); + const rows = await engine.executeRaw<{ fm: Record }>( + `SELECT frontmatter AS fm FROM pages WHERE slug = 'wiki/originals/ideas/2026-07-17-raw-src-def456'`, + ); + const fm = rows[0].fm as Record; + expect(fm.dream_generated).toBe(true); + expect(fm.raw_source).toBe('/transcripts/2026-07-17-standup.md'); + }); }); diff --git a/test/doctor-raw-provenance.test.ts b/test/doctor-raw-provenance.test.ts new file mode 100644 index 000000000..93397d56f --- /dev/null +++ b/test/doctor-raw-provenance.test.ts @@ -0,0 +1,110 @@ +/** + * #1978 — raw-source persistence guarantee (warn-only v1). + * + * `rawProvenanceCheck` flags synthesized/derived pages (dream_generated:true + * frontmatter or type:synthesis) that carry NO raw trace (raw_trace / + * raw_source / source_uri frontmatter, attached raw_data row, or + * synthesis_evidence rows) and NO explicit raw_trace_exempt marker. + * + * Runs against real PGLite so the SQL shape (`?|` key-existence operator + + * NOT EXISTS subqueries) is pinned on an actual engine, not a mock. + */ + +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { PGLiteEngine } from '../src/core/pglite-engine.ts'; +import { rawProvenanceCheck } from '../src/commands/doctor.ts'; +import { categorizeCheck } from '../src/core/doctor-categories.ts'; +import type { BrainEngine } from '../src/core/engine.ts'; + +let engine: PGLiteEngine; + +beforeAll(async () => { + engine = new PGLiteEngine(); + await engine.connect({}); + await engine.initSchema(); +}); + +afterAll(async () => { + await engine.disconnect(); +}); + +describe('rawProvenanceCheck (#1978, warn-only v1)', () => { + test('empty brain → ok', async () => { + const result = await rawProvenanceCheck(engine as unknown as BrainEngine); + expect(result.name).toBe('raw_provenance'); + expect(result.status).toBe('ok'); + }); + + test('flags only the synthesized page without a trace; every trace/exemption shape passes', async () => { + // 1. VIOLATION: dream-generated, no trace, no exemption. + await engine.putPage('wiki/derived/no-trace', { + type: 'note', title: 'No trace', compiled_truth: 'body', timeline: '', + frontmatter: { dream_generated: true }, + }); + // 2. OK: dream-generated with raw_source frontmatter. + await engine.putPage('wiki/derived/with-raw-source', { + type: 'note', title: 'Has raw_source', compiled_truth: 'body', timeline: '', + frontmatter: { dream_generated: true, raw_source: '/transcripts/2026-07-01.md' }, + }); + // 3. OK: type:synthesis with explicit exemption. + await engine.putPage('synthesis/exempt-page', { + type: 'synthesis', title: 'Exempt', compiled_truth: 'body', timeline: '', + frontmatter: { raw_trace_exempt: true, raw_trace_exempt_reason: 'test' }, + }); + // 4. OK: hand-authored note — not synthesized, never flagged. + await engine.putPage('wiki/hand-authored', { + type: 'note', title: 'Hand authored', compiled_truth: 'body', timeline: '', + frontmatter: {}, + }); + // 5. OK: dream-generated with an attached raw_data row. + const withRaw = await engine.putPage('wiki/derived/with-raw-data', { + type: 'note', title: 'Has raw_data', compiled_truth: 'body', timeline: '', + frontmatter: { dream_generated: true }, + }); + await engine.executeRaw( + `INSERT INTO raw_data (page_id, source, data) VALUES ($1, 'test', '{}'::jsonb)`, + [withRaw.id], + ); + + const result = await rawProvenanceCheck(engine as unknown as BrainEngine); + expect(result.status).toBe('warn'); + expect(result.message).toContain('1 synthesized page(s)'); + expect(result.message).toContain('wiki/derived/no-trace'); + expect(result.message).not.toContain('with-raw-source'); + expect(result.message).not.toContain('exempt-page'); + expect(result.message).not.toContain('hand-authored'); + expect(result.message).not.toContain('with-raw-data'); + }); + + test('stamping an exemption on the violator clears the warning', async () => { + await engine.executeRaw( + `UPDATE pages SET frontmatter = frontmatter || '{"raw_trace_exempt": true, "raw_trace_exempt_reason": "reviewed"}'::jsonb + WHERE slug = 'wiki/derived/no-trace'`, + ); + const result = await rawProvenanceCheck(engine as unknown as BrainEngine); + expect(result.status).toBe('ok'); + }); + + test('soft-deleted violators are not flagged', async () => { + await engine.putPage('wiki/derived/deleted-no-trace', { + type: 'note', title: 'Deleted violator', compiled_truth: 'body', timeline: '', + frontmatter: { dream_generated: true }, + }); + expect((await rawProvenanceCheck(engine as unknown as BrainEngine)).status).toBe('warn'); + await engine.executeRaw( + `UPDATE pages SET deleted_at = now() WHERE slug = 'wiki/derived/deleted-no-trace'`, + ); + expect((await rawProvenanceCheck(engine as unknown as BrainEngine)).status).toBe('ok'); + }); + + test('query failure degrades to warn, never throws', async () => { + const broken = { executeRaw: async () => { throw new Error('boom'); } } as unknown as BrainEngine; + const result = await rawProvenanceCheck(broken); + expect(result.status).toBe('warn'); + expect(result.message).toContain('Could not check'); + }); + + test('raw_provenance is categorized as a brain check', () => { + expect(categorizeCheck('raw_provenance')).toBe('brain'); + }); +}); diff --git a/test/extract/receipt-writer.test.ts b/test/extract/receipt-writer.test.ts index d2a52a7c6..3de8ac76e 100644 --- a/test/extract/receipt-writer.test.ts +++ b/test/extract/receipt-writer.test.ts @@ -115,6 +115,11 @@ describe('writeReceipt — frontmatter D-EXTRACT-19 belt+suspenders', () => { // belt + suspenders: both anti-loop flags are present expect(page.frontmatter?.type).toBe('extract_receipt'); expect(page.frontmatter?.dream_generated).toBe(true); + // #1978: receipts are operation records, not derived documents — + // explicit raw-trace exemption so the doctor raw_provenance check + // (warn-only v1) stays quiet. + expect(page.frontmatter?.raw_trace_exempt).toBe(true); + expect(typeof page.frontmatter?.raw_trace_exempt_reason).toBe('string'); }); test('stamps optional model_id + eval_pass + eval_score when supplied', async () => { From eb6cb4a16f7674d2c1d74ec3e989282d998bc95d Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:42:55 -0700 Subject: [PATCH 17/39] fix(cycle): extract_atoms stamps concepts so synthesize_concepts has material (#2123) (#2124) (#3308) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit synthesize-concepts.ts's design comment says extract_atoms stamps a `concepts:` frontmatter field on each atom and :92 consumes ONLY that field — but the extractor never wrote it, so the atoms → concepts pipeline was dead end-to-end: every cycle reported "synthesize_concepts: skipped — no atoms with concept refs" no matter how many atoms accumulated (696 page-derived atoms / 0 with concepts on our production brain before an external backfill). Fix, all on the extractor side (no synthesize change needed): - EXTRACT_PROMPT asks for `concepts` (1-3 kebab-case TOPIC labels) with an explicit reuse-over-coinage instruction — labels must cluster, since synthesize_concepts only materializes groups of >=2. - parseAtomsResponse validates labels (kebab regex, max 3, drop invalid; empty -> undefined). - The putPage frontmatter write stamps `concepts` alongside lesson / source_quote. Tests: 4 parse cases + an end-to-end regression that goes extractor -> real frontmatter -> synthesize_concepts' OWN DB query path -> concept page. The existing tests fed synthesize via the `_atoms` seam, which is exactly how this gap survived. Validated in production ahead of this PR by stamping the same shape externally: the next synthesize_concepts run wrote 33 concept pages (T2=7/T3=26) from 60 stamped atoms, zero failures. Co-authored-by: 陈源泉 <84364275+ChenyqThu@users.noreply.github.com> Co-authored-by: 陈源泉 Co-authored-by: Claude Fable 5 --- src/core/cycle/extract-atoms.ts | 29 +++++++- .../extract-atoms-synthesize-concepts.test.ts | 66 +++++++++++++++++++ 2 files changed, 92 insertions(+), 3 deletions(-) diff --git a/src/core/cycle/extract-atoms.ts b/src/core/cycle/extract-atoms.ts index 0b12b4d79..49dc9b749 100644 --- a/src/core/cycle/extract-atoms.ts +++ b/src/core/cycle/extract-atoms.ts @@ -171,10 +171,20 @@ interface ExtractedAtom { body: string; source_quote?: string; lesson?: string; + /** + * 1-3 kebab-case topic labels for concept clustering. Consumed by + * synthesize_concepts (groups atoms by `frontmatter.concepts`; only + * labels shared by >=2 atoms materialize a concept page, so the prompt + * biases reuse-over-coinage). #2123. + */ + concepts?: string[]; virality_score?: number; emotional_register?: string; } +/** kebab-case validator for concept labels ("captive-portal", "channel-pricing"). */ +const CONCEPT_LABEL_RE = /^[a-z0-9]+(-[a-z0-9]+)*$/; + const EXTRACT_PROMPT = `You extract atomic content nuggets from a transcript. An atom is a single-source, self-contained idea that could become a tweet, @@ -185,12 +195,17 @@ quote, or short essay angle. Each atom must: Output a JSON array of atoms (1-3 per transcript, never more than 3). Each atom: {title (≤80 chars), atom_type, body (2-4 sentences), -source_quote (verbatim ≤200 chars), lesson (one sentence), virality_score -(0-100), emotional_register (one of: shocking, inspiring, funny, sobering, -practical, controversial)}. +source_quote (verbatim ≤200 chars), lesson (one sentence), concepts +(1-3 topic labels), virality_score (0-100), emotional_register (one of: +shocking, inspiring, funny, sobering, practical, controversial)}. atom_type MUST be one of: ${ATOM_TYPES.join(', ')}. +concepts are kebab-case English TOPIC labels used to cluster atoms into +concept pages (e.g. "captive-portal", "channel-pricing-strategy") — never +entity or brand names. Use the same label for the same topic across atoms; +prefer a label you already used over coining a near-synonym. + Output ONLY the JSON array, no prose.`; interface DiscoveredPage { @@ -600,6 +615,7 @@ export async function runPhaseExtractAtoms( source_hash: item.contentHash.slice(0, 16), ...(atom.source_quote && { source_quote: atom.source_quote }), ...(atom.lesson && { lesson: atom.lesson }), + ...(atom.concepts && atom.concepts.length > 0 && { concepts: atom.concepts }), ...(atom.virality_score !== undefined && { virality_score: atom.virality_score }), ...(atom.emotional_register && { emotional_register: atom.emotional_register }), extracted_at: new Date().toISOString(), @@ -736,6 +752,13 @@ export function parseAtomsResponse(raw: string): ExtractedAtom[] { body, source_quote: typeof obj.source_quote === 'string' ? obj.source_quote.slice(0, 500) : undefined, lesson: typeof obj.lesson === 'string' ? obj.lesson : undefined, + concepts: (() => { + if (!Array.isArray(obj.concepts)) return undefined; + const labels = obj.concepts + .filter((c): c is string => typeof c === 'string' && CONCEPT_LABEL_RE.test(c)) + .slice(0, 3); + return labels.length > 0 ? labels : undefined; + })(), virality_score: typeof obj.virality_score === 'number' && obj.virality_score >= 0 && diff --git a/test/cycle/extract-atoms-synthesize-concepts.test.ts b/test/cycle/extract-atoms-synthesize-concepts.test.ts index 4b410fa9a..8b409568c 100644 --- a/test/cycle/extract-atoms-synthesize-concepts.test.ts +++ b/test/cycle/extract-atoms-synthesize-concepts.test.ts @@ -370,3 +370,69 @@ describe('v0.41 T6: runPhaseSynthesizeConcepts via stubbed chat', () => { expect((page[0].fm as Record).tier).toBe('T1'); }); }); + +// #2123 — extract_atoms must stamp `concepts` so synthesize_concepts has +// material. The pre-fix pipeline was broken end-to-end: the extractor +// never wrote the field, and every synthesize_concepts cycle skipped with +// "no atoms with concept refs". The earlier describe blocks feed +// synthesize via the `_atoms` seam, which is exactly how the gap survived +// — so the last test here goes extractor → REAL frontmatter → real DB +// query path → concept page. +describe('#2123: concepts label parsing', () => { + test('keeps valid kebab-case labels', () => { + const raw = `[{"title":"T","atom_type":"insight","body":"b","concepts":["captive-portal","tls-certificates"]}]`; + expect(parseAtomsResponse(raw)[0].concepts).toEqual(['captive-portal', 'tls-certificates']); + }); + + test('filters non-kebab labels, keeps the rest', () => { + const raw = `[{"title":"T","atom_type":"insight","body":"b","concepts":["Captive Portal","tp_link","UPPER","valid-label"]}]`; + expect(parseAtomsResponse(raw)[0].concepts).toEqual(['valid-label']); + }); + + test('truncates to 3 labels', () => { + const raw = `[{"title":"T","atom_type":"insight","body":"b","concepts":["a","b","c","d","e"]}]`; + expect(parseAtomsResponse(raw)[0].concepts).toEqual(['a', 'b', 'c']); + }); + + test('absent / non-array / all-invalid → undefined', () => { + expect(parseAtomsResponse(`[{"title":"T","atom_type":"insight","body":"b"}]`)[0].concepts).toBeUndefined(); + expect(parseAtomsResponse(`[{"title":"T","atom_type":"insight","body":"b","concepts":"not-an-array"}]`)[0].concepts).toBeUndefined(); + expect(parseAtomsResponse(`[{"title":"T","atom_type":"insight","body":"b","concepts":["Bad Label!"]}]`)[0].concepts).toBeUndefined(); + }); +}); + +describe('#2123: extractor stamps concepts → synthesize_concepts consumes via real DB path', () => { + test('end-to-end: atoms with shared label materialize a concept page', async () => { + const chat = stubChat(`[ + {"title":"Cert warning on guest wifi","atom_type":"insight","body":"Portal redirects to an IP-based HTTPS URL.","concepts":["captive-portal"]}, + {"title":"iPhone portal popup is flaky","atom_type":"critique","body":"CNA probe behavior differs across iOS versions.","concepts":["captive-portal"]} + ]`); + const extract = await runPhaseExtractAtoms(engine, { + _transcripts: [{ filePath: '/fake/notes.txt', content: 'content', contentHash: 'cc2123' }], + _pages: [], + _chat: chat, + }); + expect(extract.status).toBe('ok'); + expect(extract.details?.atoms_extracted).toBe(2); + + // Frontmatter really carries the label (a jsonb array, not a string). + const stamped = await engine.executeRaw<{ concepts: unknown }>( + `SELECT frontmatter->'concepts' AS concepts FROM pages WHERE type = 'atom'`, + ); + expect(stamped.length).toBe(2); + for (const row of stamped) { + const arr = typeof row.concepts === 'string' ? JSON.parse(row.concepts) : row.concepts; + expect(arr).toEqual(['captive-portal']); + } + + // NO _atoms seam: synthesize discovers the atoms through its own + // DB query — this is the path that was dead before the fix. + const synth = await runPhaseSynthesizeConcepts(engine, { _chat: stubChat('unused — T3 is deterministic') }); + expect(synth.status).toBe('ok'); + expect(synth.details?.concepts_written).toBe(1); + const concept = await engine.executeRaw<{ slug: string }>( + `SELECT slug FROM pages WHERE slug = 'concepts/captive-portal' AND type = 'concept'`, + ); + expect(concept.length).toBe(1); + }); +}); From 38b8b1e41eefe3fd3482b59af0dc34b24fb6e8e7 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:43:00 -0700 Subject: [PATCH 18/39] fix(doctor): stop claiming "Brain is at target" when the target is unreachable (#2151) (#3339) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `gbrain doctor --remediation-plan` printed two consecutive lines that contradicted each other when the brain was below target AND the target was unreachable with autonomous remediation: Brain score: 45/100 → target 90 Target unreachable: max with autonomous remediation is 70/100. No remediations needed. Brain is at target. The second sentence hid the real next step (configure the prereqs that would lift `max_reachable_score`) and made the brain look healthy when it was not. Fix: gate the "Brain is at target" line on `brain_score_current >= targetScore`. When the plan is empty AND the brain is below target, the "Target unreachable" line above is already the user-facing explanation; the `Blocked checks` block below surfaces the manual gap. Extracted `renderRemediationPlanLines(plan, targetScore): string[]` as a pure helper alongside `runRemediationPlan` so the regression coverage asserts on the rendered output directly rather than mocking `console.log`. `runRemediationPlan` now joins the lines verbatim through console.log; behavior is byte-identical for every case other than the fixed contradiction. Five regression tests cover: unreachable-and-below-target (the bug case), reachable-and-at-target, exact-target, below-target-with-plan, unreachable-with-partial-plan. 38 tests across the adjacent doctor test files stay green; `bun run typecheck` clean. Co-authored-by: Brett --- src/commands/doctor.ts | 60 ++++++++++-- test/doctor-remediation-plan-render.test.ts | 103 ++++++++++++++++++++ 2 files changed, 155 insertions(+), 8 deletions(-) create mode 100644 test/doctor-remediation-plan-render.test.ts diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index 5b917f9fa..42a167684 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -7924,27 +7924,71 @@ export async function runRemediationPlan( return; } - // Human output - console.log(`Brain score: ${plan.brain_score_current}/100 → target ${targetScore}`); + for (const line of renderRemediationPlanLines(plan, targetScore)) { + console.log(line); + } +} + +/** + * Human-render the remediation plan into a sequence of console lines. + * Exported for unit-test access — `runRemediationPlan` consumes it + * verbatim and only adds the JSON-mode short-circuit. + * + * Gating the "at target" line on `brain_score_current >= targetScore` + * is load-bearing: when the plan is empty AND the target is unreachable, + * the prior shape printed both "Target unreachable: …" and "Brain is at + * target" back-to-back, which contradicted itself and hid the real next + * step (manual prereq config to lift `max_reachable_score`). + */ +export function renderRemediationPlanLines( + plan: RemediationPlanShape, + targetScore: number, +): string[] { + const lines: string[] = []; + lines.push(`Brain score: ${plan.brain_score_current}/100 → target ${targetScore}`); if (plan.target_unreachable) { - console.log(`Target unreachable: max with autonomous remediation is ${plan.max_reachable_score}/100.`); + lines.push(`Target unreachable: max with autonomous remediation is ${plan.max_reachable_score}/100.`); } if (plan.plan.length === 0) { - console.log('No remediations needed. Brain is at target.'); + if (plan.brain_score_current >= targetScore) { + lines.push('No remediations needed. Brain is at target.'); + } + // When brain_score < targetScore and plan is empty, the unreachable + // line (if applicable) is the user-facing explanation; the blocked- + // checks block below surfaces the manual gap. Don't follow with a + // misleading "at target" claim. } else { - console.log(`Plan: ${plan.plan.length} step(s), est ${plan.est_total_seconds}s, est $${plan.est_total_usd_cost.toFixed(2)}`); + lines.push(`Plan: ${plan.plan.length} step(s), est ${plan.est_total_seconds}s, est $${plan.est_total_usd_cost.toFixed(2)}`); for (const step of plan.plan) { const protectedMark = step.protected ? ' [PROTECTED]' : ''; const costMark = step.est_usd_cost ? ` ($${step.est_usd_cost.toFixed(2)})` : ''; - console.log(` ${step.step}. [${step.severity}] ${step.job}${protectedMark} — ${step.rationale}${costMark}`); + lines.push(` ${step.step}. [${step.severity}] ${step.job}${protectedMark} — ${step.rationale}${costMark}`); } } if (plan.blocked.length > 0) { - console.log(`\nBlocked checks (prereq missing):`); + lines.push(`\nBlocked checks (prereq missing):`); for (const b of plan.blocked) { - console.log(` - ${b.check}: ${b.reason}`); + lines.push(` - ${b.check}: ${b.reason}`); } } + return lines; +} + +interface RemediationPlanShape { + brain_score_current: number; + target_unreachable: boolean; + max_reachable_score: number; + plan: Array<{ + step: number; + severity: string; + job: string; + protected?: boolean; + est_usd_cost?: number; + rationale: string; + }>; + est_total_seconds: number; + est_total_usd_cost: number; + blocked: Array<{ check: string; reason: string }>; } /** diff --git a/test/doctor-remediation-plan-render.test.ts b/test/doctor-remediation-plan-render.test.ts new file mode 100644 index 000000000..b73d314ff --- /dev/null +++ b/test/doctor-remediation-plan-render.test.ts @@ -0,0 +1,103 @@ +// Regression coverage for the `gbrain doctor --remediation-plan` verdict +// contradiction: when the brain was below target AND the target was +// unreachable, the human renderer printed "Target unreachable: max with +// autonomous remediation is N/100" followed immediately by "No +// remediations needed. Brain is at target." — two consecutive lines that +// contradicted each other and hid the real next step. + +import { describe, test, expect } from 'bun:test'; +import { renderRemediationPlanLines } from '../src/commands/doctor.ts'; + +type Plan = Parameters[0]; + +function planFixture(overrides: Partial): Plan { + return { + brain_score_current: 0, + target_unreachable: false, + max_reachable_score: 100, + plan: [], + est_total_seconds: 0, + est_total_usd_cost: 0, + blocked: [], + ...overrides, + }; +} + +describe('renderRemediationPlanLines', () => { + test('unreachable + brain below target — never claims "Brain is at target"', () => { + const plan = planFixture({ + brain_score_current: 45, + target_unreachable: true, + max_reachable_score: 70, + plan: [], + blocked: [{ check: 'link_density', reason: 'no enrichment keys configured' }], + }); + const text = renderRemediationPlanLines(plan, 90).join('\n'); + expect(text).toContain('Brain score: 45/100'); + expect(text).toContain('Target unreachable: max with autonomous remediation is 70/100'); + expect(text).not.toContain('Brain is at target'); + expect(text).toContain('Blocked checks'); + }); + + test('reachable, brain at or above target, no plan — emits the "at target" line', () => { + const plan = planFixture({ + brain_score_current: 95, + target_unreachable: false, + max_reachable_score: 100, + plan: [], + }); + const text = renderRemediationPlanLines(plan, 90).join('\n'); + expect(text).toContain('Brain is at target'); + expect(text).not.toContain('Target unreachable'); + }); + + test('brain at exact target with empty plan — still "at target"', () => { + const plan = planFixture({ + brain_score_current: 90, + target_unreachable: false, + plan: [], + }); + const text = renderRemediationPlanLines(plan, 90).join('\n'); + expect(text).toContain('Brain is at target'); + }); + + test('brain below target with plan steps — lists the plan, no "at target" line', () => { + const plan = planFixture({ + brain_score_current: 60, + target_unreachable: false, + max_reachable_score: 100, + est_total_seconds: 120, + est_total_usd_cost: 0.4, + plan: [ + { step: 1, severity: 'high', job: 'embed-coverage', rationale: 'missing embeddings' }, + { step: 2, severity: 'med', job: 'consolidate', rationale: 'pending entity merges', est_usd_cost: 0.4 }, + ], + }); + const lines = renderRemediationPlanLines(plan, 90); + const text = lines.join('\n'); + expect(text).toContain('Plan: 2 step(s)'); + expect(text).toContain('1. [high] embed-coverage'); + expect(text).toContain('2. [med] consolidate'); + expect(text).toContain('($0.40)'); + expect(text).not.toContain('Brain is at target'); + }); + + test('unreachable but a partial plan exists — plan prints, "at target" suppressed', () => { + const plan = planFixture({ + brain_score_current: 30, + target_unreachable: true, + max_reachable_score: 55, + est_total_seconds: 90, + est_total_usd_cost: 0.2, + plan: [ + { step: 1, severity: 'high', job: 'embed-coverage', rationale: 'reach max_reachable' }, + ], + blocked: [{ check: 'enrichment', reason: 'no provider key configured' }], + }); + const text = renderRemediationPlanLines(plan, 90).join('\n'); + expect(text).toContain('Target unreachable: max with autonomous remediation is 55/100'); + expect(text).toContain('Plan: 1 step(s)'); + expect(text).toContain('Blocked checks'); + expect(text).not.toContain('Brain is at target'); + }); +}); From 26c6bad44565da296a9a52410810df573d37bc56 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:43:06 -0700 Subject: [PATCH 19/39] Reject unknown init flags before migrations (#2201) (#3307) Co-authored-by: caioribeiroclw-pixel --- src/commands/init.ts | 61 ++++++++++++++++++++++++++++++++++ test/init-migrate-only.test.ts | 19 +++++++++++ 2 files changed, 80 insertions(+) diff --git a/src/commands/init.ts b/src/commands/init.ts index 14e33f6cf..3a2ff7e08 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -26,6 +26,8 @@ export async function runInit(args: string[]) { return; } + validateInitFlags(args); + const isSupabase = args.includes('--supabase'); const isPGLite = args.includes('--pglite'); const isMcpOnly = args.includes('--mcp-only'); @@ -151,6 +153,65 @@ export async function runInit(args: string[]) { return initPostgres({ databaseUrl, jsonOutput, apiKey, aiOpts, schemaPack, skipEmbedCheck }); } +const INIT_BOOLEAN_FLAGS = new Set([ + '--pglite', + '--supabase', + '--mcp-only', + '--force', + '--non-interactive', + '--migrate-only', + '--json', + '--no-embedding', + '--skip-embed-check', +]); + +const INIT_VALUE_FLAGS = new Set([ + '--url', + '--key', + '--path', + '--schema-pack', + '--embedding-model', + '--model', + '--embedding-dimensions', + '--expansion-model', + '--chat-model', + '--mcp-url', + '--issuer-url', + '--oauth-client-id', + '--oauth-client-secret', +]); + +function validateInitFlags(args: string[]) { + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + if (!arg.startsWith('-')) continue; + + if (INIT_BOOLEAN_FLAGS.has(arg)) continue; + + if (INIT_VALUE_FLAGS.has(arg)) { + if (i + 1 >= args.length || args[i + 1].startsWith('-')) { + failInitFlag(`gbrain init: ${arg} requires a value`, args.includes('--json')); + } + i += 1; + continue; + } + + if (arg.startsWith('--')) { + failInitFlag(`gbrain init: unknown flag ${arg}`, args.includes('--json')); + } + } +} + +function failInitFlag(message: string, jsonOutput: boolean): never { + if (jsonOutput) { + console.log(JSON.stringify({ status: 'error', reason: 'invalid_flag', message })); + } else { + console.error(message); + console.error('Run `gbrain init --help` for supported flags.'); + } + process.exit(1); +} + interface ResolveAIOptionsArgs { verbose: string | null; // --embedding-model shorthand: string | null; // --model diff --git a/test/init-migrate-only.test.ts b/test/init-migrate-only.test.ts index 2f06001ec..a3732e5f1 100644 --- a/test/init-migrate-only.test.ts +++ b/test/init-migrate-only.test.ts @@ -57,6 +57,25 @@ afterEach(() => { }); describe('gbrain init --migrate-only — error paths', () => { + test('rejects unknown flags before any migrate-only side effects', () => { + const result = run(['init', '--migrate-only', '--dry-run']); + expect(result.exitCode).toBe(1); + expect(result.stderr).toContain('unknown flag --dry-run'); + // Unknown safety flags must not fall through to the migration path. + expect(result.stderr).not.toContain('No brain configured'); + expect(existsSync(join(tmp, '.gbrain', 'config.json'))).toBe(false); + }); + + test('unknown flags respect --json output', () => { + const result = run(['init', '--migrate-only', '--dry-run', '--json']); + expect(result.exitCode).toBe(1); + const lines = result.stdout.split('\n').filter((l: string) => l.trim().startsWith('{')); + const parsed = JSON.parse(lines[lines.length - 1]); + expect(parsed.status).toBe('error'); + expect(parsed.reason).toBe('invalid_flag'); + expect(parsed.message).toContain('unknown flag --dry-run'); + }); + test('errors with clear message when no config exists', () => { const result = run(['init', '--migrate-only']); expect(result.exitCode).toBe(1); From b313938e865a58533b4fb411ab95b4a71b2b03ee Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:43:11 -0700 Subject: [PATCH 20/39] fix(queue): dead/cancelled jobs no longer block idempotency re-submission (#2253) (#3306) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit queue.add() with an idempotency_key returns any existing row regardless of status. This means dead jobs (exhausted retries from a transient provider outage) permanently block re-submission of the same work — even after the underlying issue is fixed. Fix: when the existing row is dead or cancelled, NULL its idempotency_key (preserving the row for audit) and fall through to the INSERT path so a fresh job can be created. Affects dream synthesize children that died during provider migrations (429 rate-limit on old Anthropic proxy, tool-results-missing on old OpenRouter). 45 dead children were blocking re-synthesis of transcripts in production. Includes 4 new tests covering dead, cancelled, completed, and active status interactions with idempotency dedup. Co-authored-by: Rafael Reis <57492577+rafaelreis-r@users.noreply.github.com> Co-authored-by: Rafael Reis --- src/core/minions/queue.ts | 17 +++++++++- test/minions.test.ts | 67 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/src/core/minions/queue.ts b/src/core/minions/queue.ts index ccf71cd96..0d0780fdb 100644 --- a/src/core/minions/queue.ts +++ b/src/core/minions/queue.ts @@ -133,12 +133,27 @@ export class MinionQueue { // 1. Idempotency fast path — if a row already exists for this key, return it // without doing any other work. The unique partial index guarantees // no second row can be inserted with the same non-null key. + // + // Dead/cancelled jobs represent permanently-failed work whose + // idempotency slot must be freed so a fresh attempt can be inserted. + // We NULL the key (preserving the row for audit) and fall through + // to the INSERT path below. if (opts?.idempotency_key) { const existing = await tx.executeRaw>( `SELECT * FROM minion_jobs WHERE idempotency_key = $1`, [opts.idempotency_key] ); - if (existing.length > 0) return rowToMinionJob(existing[0]); + if (existing.length > 0) { + const existingJob = rowToMinionJob(existing[0]); + if (existingJob.status === 'dead' || existingJob.status === 'cancelled') { + await tx.executeRaw( + `UPDATE minion_jobs SET idempotency_key = NULL WHERE id = $1`, + [existingJob.id] + ); + } else { + return existingJob; + } + } } // 1b. Submission-time backpressure for high-frequency named jobs. diff --git a/test/minions.test.ts b/test/minions.test.ts index 3f6bf3c07..0909d7e44 100644 --- a/test/minions.test.ts +++ b/test/minions.test.ts @@ -1582,6 +1582,73 @@ describe('MinionQueue: Idempotency', () => { expect(j2.id).toBe(j1.id); expect(j2.data).toEqual({ v: 1 }); // first wins }); + + test('dead job with idempotency_key allows re-submission', async () => { + const j1 = await queue.add('test-synth', { prompt: 'synthesize' }, { + idempotency_key: 'dream:synth:test:abc123', + max_attempts: 1, + }); + await engine.executeRaw( + `UPDATE minion_jobs SET status = 'dead', finished_at = now() WHERE id = $1`, + [j1.id] + ); + const j2 = await queue.add('test-synth', { prompt: 'synthesize' }, { + idempotency_key: 'dream:synth:test:abc123', + max_attempts: 8, + }); + expect(j2.id).not.toBe(j1.id); + expect(j2.status).toBe('waiting'); + const oldRow = await engine.executeRaw<{ idempotency_key: string | null }>( + `SELECT idempotency_key FROM minion_jobs WHERE id = $1`, + [j1.id] + ); + expect(oldRow[0].idempotency_key).toBeNull(); + }); + + test('cancelled job with idempotency_key allows re-submission', async () => { + const j1 = await queue.add('test-synth', {}, { + idempotency_key: 'dream:synth:test:cancel', + }); + await engine.executeRaw( + `UPDATE minion_jobs SET status = 'cancelled', finished_at = now() WHERE id = $1`, + [j1.id] + ); + const j2 = await queue.add('test-synth', {}, { + idempotency_key: 'dream:synth:test:cancel', + }); + expect(j2.id).not.toBe(j1.id); + expect(j2.status).toBe('waiting'); + }); + + test('completed job with idempotency_key still blocks re-submission', async () => { + const j1 = await queue.add('sync', {}, { + idempotency_key: 'dream:synth:test:completed', + }); + await engine.executeRaw( + `UPDATE minion_jobs SET status = 'completed', finished_at = now() WHERE id = $1`, + [j1.id] + ); + const j2 = await queue.add('sync', {}, { + idempotency_key: 'dream:synth:test:completed', + }); + expect(j2.id).toBe(j1.id); + expect(j2.status).toBe('completed'); + }); + + test('active job with idempotency_key still blocks re-submission', async () => { + const j1 = await queue.add('sync', {}, { + idempotency_key: 'dream:synth:test:active', + }); + await engine.executeRaw( + `UPDATE minion_jobs SET status = 'active' WHERE id = $1`, + [j1.id] + ); + const j2 = await queue.add('sync', {}, { + idempotency_key: 'dream:synth:test:active', + }); + expect(j2.id).toBe(j1.id); + expect(j2.status).toBe('active'); + }); }); // --- v7 child_done auto-post --- From eba9680775dcafacdd6c66bd578e957e4254e951 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:00:56 -0700 Subject: [PATCH 21/39] feat(ai): claude-cli recipe for native gateway-based subagent dispatch (#2277) (#3310) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(subagent): claude-cli MessagesClient adapter (baseline, no tool use) Closes #334 (partially — text-only baseline; tool use lands in the next commit on this branch). Adds a MessagesClient adapter that shells out to `claude --print --output-format json --model ` instead of the Anthropic SDK. When `GBRAIN_USE_CLAUDE_CLI=1` is set, the subagent worker registers the adapter in place of the SDK client; the default path (Anthropic SDK with ANTHROPIC_API_KEY) is unchanged when the env var is unset or set to anything else. The benefit is that Claude Max subscribers can run Minions subagents against their existing OAuth subscription, no ANTHROPIC_API_KEY needed. New: src/core/minions/handlers/claude-cli-adapter.ts - Implements the MessagesClient interface exported from subagent.ts. - Strips provider prefixes (`anthropic:`, `litellm:`) from the model id because `claude --print` only accepts CLI-native aliases (`sonnet`, `opus`, `haiku`, or the bare `claude-*-N-M` form). - Flattens the Anthropic messages array into a single text prompt for claude-cli stdin. Tool blocks (tool_use / tool_result) are stringified as placeholders so multi-turn conversations stay coherent in this baseline; native tool_use round-tripping is the follow-up commit. - Spawns claude with stdio piped, captures stdout, parses the `{type:"result", subtype:"success", result, usage, ...}` JSON envelope, and returns it as a properly shaped Anthropic.Message with `stop_reason: 'end_turn'`. - Token totals propagate from the claude usage block so the subagent handler's `ctx.updateTokens()` reports usable numbers. - AbortSignal is wired through to SIGTERM the child so the subagent loop's cancellation path stays correct. Modified: src/commands/jobs.ts (worker registration) - Conditionally constructs a MessagesClient via the new adapter when GBRAIN_USE_CLAUDE_CLI=1. - Passes it into makeSubagentHandler({ engine, client: subagentClient }). - Logs `[minion worker] subagent routing via claude-cli (GBRAIN_USE_CLAUDE_CLI=1)` on startup so the env var status is operator-visible. Limitations of this commit (addressed in the follow-up): - Tool use is not yet supported. Tools in params.tools are ignored; the adapter returns a single text block with stop_reason='end_turn'. - Token counts come from claude-cli's reporting and may not match the Anthropic API's accounting precisely (especially for cache tiers). Original design from #334; this commit preserves that author's attribution. The follow-up commits on this branch carry the tool-use implementation. * feat(subagent): tool use + context isolation + convention rename on top of jarvisdoes baseline Builds on the previous commit (jarvisdoes's #334 baseline) by adding three things the upstream issue called out as gaps or that surfaced during review: 1. Tool use support via system-prompt-instructed JSON emission. 2. Context isolation flags so claude-cli does not load operator-level CLAUDE.md, skills, and local project context into every subagent call. 3. Env var rename from GBRAIN_USE_CLAUDE_CLI=1 to GBRAIN_SUBAGENT_PROVIDER=claude-cli to match the existing GBRAIN__= convention used by GBRAIN_CHAT_MODEL, GBRAIN_EMBEDDING_MODEL, GBRAIN_EXPANSION_MODEL. ## Tool use The MessagesClient interface returns Anthropic.Message objects whose content array may include tool_use blocks. The subagent handler filters those blocks and dispatches each tool, so any backend that produces correctly shaped tool_use blocks gets the same loop behavior as the Anthropic SDK. The adapter injects a system-prompt addendum describing the tool registry plus an emission protocol: [{"id": "...", "name": "...", "input": {...}}, ...] After the response comes back, extractToolCalls() scans for the block, parses the JSON (tolerant of optional ```json fencing), and converts each entry into a tool_use content block. Multiple parallel tool calls in one turn are supported via the array shape; this is the exact case that breaks today on the codex-proxy / litellm GPT-5.x bridge where parallel tool-call response IDs get dropped. Defensive fallbacks: - Malformed JSON inside the block: drop to text-only, stop_reason='end_turn'. - Unterminated (no close tag): drop to text-only. - Model omits id field: adapter synthesizes a toolu_claude_cli_ id. - Empty response: still hand the subagent loop a well-formed content array so the .filter chain does not crash. ## Context isolation claude-cli auto-discovers CLAUDE.md from cwd upward and injects the operator's skills + plugins + auto-memory into the default system prompt. On a real install that is ~42-65k tokens of contamination per subagent call, with both cost and behavioral consequences (the subagent picks up the operator's coding conventions, opinions, and preferences). The maximum suppression that still preserves OAuth / Claude Max subscription auth is: - Spawn from a dedicated clean cwd (tmpdir-based) so LOCAL CLAUDE.md auto-discovery has nothing to find. -13k tokens on a real gbrain install where CLAUDE.md is substantial. - --disable-slash-commands so skill resolution does not pull in /skill-name handlers. - --system-prompt so the default system prompt is replaced rather than appended to. The --bare flag would also strip user-level ~/.claude/CLAUDE.md but it forces ANTHROPIC_API_KEY auth, defeating the whole point of this adapter. The remaining ~42k cached tokens from user-level instructions are accepted as a cost-trivial trade-off because the Max subscription absorbs the per-call cost. Behavioral contamination is mitigated by gbrain's strong per-call system prompt overriding any operator-level drift. ## Env var rename Surveyed all ~140 GBRAIN_* env vars in src/. The codebase uses three patterns: GBRAIN_NO_ (negative toggles), GBRAIN__ = (routing keys), GBRAIN_ALLOW_ (permissive toggles). GBRAIN_USE_* does not appear anywhere except jarvisdoes's original commit; it would introduce a fourth pattern. GBRAIN_SUBAGENT_PROVIDER=claude-cli aligns with the routing-keys family and is value-extensible — adding codex-cli / meridian-proxy / etc. later means a new value, not a new env var. The scope ('SUBAGENT_*') is also unambiguous about which calls the toggle covers; GBRAIN_USE_CLAUDE_CLI was silent on whether it applied to all gbrain LLM calls or only the subagent path. Unknown values are rejected with a fail-fast error message naming the two valid values rather than silently falling through to the default. ## Tests New file: test/claude-cli-adapter.test.ts — 12 tests, 33 assertions: - Text-only round trip (single text block, usage propagation, end_turn). - Provider prefix stripping ('anthropic:claude-sonnet-4-6' -> 'claude-sonnet-4-6'). - Single tool_use parsing. - Multiple parallel tool calls in one block (the case that triggered the codex-proxy regression). - Fenced JSON inside block. - Model-omitted id gets synthesized to toolu_claude_cli_. - Malformed JSON falls back to text. - Unterminated block falls back to text. - AbortSignal SIGTERMs the child. - Error envelope rejected with informative message. - Non-JSON output rejected with raw-output excerpt in the error. - argv + cwd assertion: --disable-slash-commands + --system-prompt are present and cwd is the dedicated tmpdir. Tests use a POSIX shell stub at GBRAIN_CLAUDE_CLI_BIN that emits a scripted --output-format json envelope, so the suite runs without claude-cli installed and without API credits. * feat(ai): claude-cli recipe with native gateway integration (supersedes #334 baseline) Replaces the MessagesClient adapter + GBRAIN_USE_CLAUDE_CLI=1 env-var gate from the previous commit on this branch with a proper gateway recipe. The recipe path gives per-call routing as a native capability: a model string like `claude-cli:claude-sonnet-4-6` lands here while a sibling `litellm:gpt-5.4` continues through the litellm-proxy / codex-proxy path in the same worker. No global env-var switch, no agent.use_gateway_loop bypass, no MessagesClient injection at jobs.ts worker startup. The previous commit on this branch (jarvisdoes baseline) is preserved in the history for #334 authorship attribution. Its functional changes are backed out here because the recipe pattern is gbrain's established integration seam; introducing a parallel MessagesClient + env-var path would have created two routing mechanisms competing for the same job. New: src/core/ai/recipes/claude-cli.ts - Recipe declaration: id 'claude-cli', tier 'native', implementation 'claude-cli', chat-only (no embedding or expansion touchpoints). - Models: claude-opus-4-7, claude-sonnet-4-6, claude-haiku-4-5-20251001. - supports_tools and supports_subagent_loop both true. - supports_prompt_cache false because the CLI handles caching internally and does not surface cache_control via the standard control plane. - auth_env.required is the empty array because the CLI owns auth (OAuth session managed by `claude login`). - Friendly aliases mirror the `anthropic` recipe: `sonnet`, `haiku`, `opus` and the same legacy-id rewrites for back-compat with stale config strings. New: src/core/ai/providers/claude-cli-language-model.ts - ClaudeCliLanguageModel class implementing the ai-sdk LanguageModelV2 interface. - doGenerate: renders the ai-sdk prompt array into a system text + user text, injects the use_tools protocol instructions when tools are present, spawns `claude --print --output-format json --model --disable-slash-commands --system-prompt ` from a dedicated tmpdir (contamination suppression: no local CLAUDE.md auto-discovery), parses the JSON envelope, extracts blocks, and returns ai-sdk-shaped LanguageModelV2Content (text + tool-call parts with stringified-JSON input matching the V2 contract). - Tolerates fenced JSON inside use_tools blocks, malformed JSON (falls back to text), missing close tag (falls back to text), model-omitted ids (synthesizes toolu_claude_cli_). - Parallel tool calls in one block round-trip cleanly: this is the case that drops IDs on the litellm + codex-proxy bridge today. - AbortSignal SIGTERMs the child for proper cancellation. - doStream throws not-supported (gateway.toolLoop is non-streaming). Modified: src/core/ai/gateway.ts - Adds case 'claude-cli' to instantiateChat (returns ClaudeCliLanguageModel). - Adds case 'claude-cli' to instantiateExpansion (same wrapper, reserved for a future expansion touchpoint declaration). - Adds case 'claude-cli' to instantiateEmbedding (throws, no embedding model, mirrors the native-anthropic path). - Lazy require() at the call site keeps the gateway module load cheap for users who never use the claude-cli path. Modified: src/core/ai/recipes/index.ts - Registers `claudeCli` in the ALL[] array next to `anthropic`. Modified: src/core/ai/types.ts - Adds 'claude-cli' to the Implementation union so the gateway switch is exhaustive at compile time. Reverted: src/commands/jobs.ts - Drops the GBRAIN_USE_CLAUDE_CLI=1 env-var gate the prior commit added. Routing now happens at the gateway based on the model string. Deleted: src/core/minions/handlers/claude-cli-adapter.ts - The MessagesClient adapter is superseded by the recipe + LanguageModelV2 path. Two routing mechanisms competing for the same job would have forced users to reason about which one wins; the recipe is the single source of truth. New file: test/claude-cli-recipe.test.ts (16 tests, 46 assertions): - Recipe registration: getRecipe returns chat-only Recipe; aliases map short names (sonnet/haiku/opus) to canonical model ids. - Text round trip: single text content block, usage propagation, stop finish reason. - Provider prefix stripping. - Single tool-call parsing. - Multiple parallel tool calls in one block. - Fenced JSON inside the block. - Model-omitted id synthesizes toolu_claude_cli_. - Malformed JSON falls back to text + stop reason. - Unterminated block falls back to text + stop reason. - Tools offered but model declines: returns text-only with stop reason so the gateway-loop treats it as a final answer rather than wedging for tool calls that never come. - AbortSignal SIGTERMs the child. - is_error envelope rejected. - Non-JSON output rejected. - doStream throws. - argv + cwd assertion: --print, --disable-slash-commands, --system-prompt are present and cwd is the dedicated tmpdir. Tests use a POSIX shell stub at GBRAIN_CLAUDE_CLI_BIN so the suite runs without claude-cli installed and without API credits. End-to-end smoke verified against a real `claude --print --model haiku` invocation: model emitted `` block with toolu_add_001 + {"a":12,"b":30}, adapter parsed back into a `tool-call` content block, finishReason 'tool-calls'. * feat(ai/claude-cli): harden subagent isolation, env scrub, verbose + stdin robustness Four defensive fixes to the claude-cli provider so a subagent call behaves identically regardless of the host's ambient Claude Code config: - Agent isolation: pass `--tools ''` and `--strict-mcp-config` so the subprocess runs as a raw LLM with no built-in tools and no inherited user MCP servers. Without `--strict-mcp-config`, each call boots the user's MCP servers (including gbrain's own), causing recursion plus PGLite single-writer lock contention. - Env scrub: drop ANTHROPIC_API_KEY / ANTHROPIC_AUTH_TOKEN / ANTHROPIC_BASE_URL from the child env so the CLI authenticates via its own OAuth subscription session. An inherited API key silently flips billing to per-token API usage, the exact setup this recipe exists to replace. - Verbose-mode compat: with `"verbose": true` in ~/.claude/settings.json, `--print --output-format json` emits an event array instead of a bare result object. Tolerate both shapes and select the result event. - stdin robustness: handle the child stdin 'error' event and wrap write/end so a missing binary (ENOENT) or early child death (EPIPE) rejects cleanly instead of crashing the worker with an unhandled error. Adds unit coverage for the env scrub, the isolation argv, and the verbose event array. Verified against claude CLI 2.1.x. * test(ai/claude-cli): cover verbose-array no-result + missing-binary reject paths Two error branches in the hardened claude-cli provider had no coverage: the verbose event-array path when no result event is present, and a missing binary surfacing as a clean spawn-failed rejection. The missing-binary case is the deterministic form of the stdin/EPIPE robustness; a synchronous stdin-write throw is not reliably triggerable in a unit test, so the real ENOENT path the handlers defend is exercised instead. Both reuse the existing shell-stub harness. --------- Co-authored-by: Brett Co-authored-by: jarvisdoes <258486803+jarvisdoes@users.noreply.github.com> Co-authored-by: Marco Maldonado <34176133+loweaxerium@users.noreply.github.com> --- src/core/ai/gateway.ts | 22 + .../ai/providers/claude-cli-language-model.ts | 444 +++++++++++++++ src/core/ai/recipes/claude-cli.ts | 71 +++ src/core/ai/recipes/index.ts | 2 + src/core/ai/types.ts | 3 +- test/claude-cli-recipe.test.ts | 535 ++++++++++++++++++ 6 files changed, 1076 insertions(+), 1 deletion(-) create mode 100644 src/core/ai/providers/claude-cli-language-model.ts create mode 100644 src/core/ai/recipes/claude-cli.ts create mode 100644 test/claude-cli-recipe.test.ts diff --git a/src/core/ai/gateway.ts b/src/core/ai/gateway.ts index 7b814a54a..dff16700a 100644 --- a/src/core/ai/gateway.ts +++ b/src/core/ai/gateway.ts @@ -1451,6 +1451,10 @@ function instantiateEmbedding(recipe: Recipe, modelId: string, cfg: AIGatewayCon throw new AIConfigError( `Anthropic has no embedding model. Use openai or google for embeddings.`, ); + case 'claude-cli': + throw new AIConfigError( + `claude-cli has no embedding model. Use openai or google for embeddings.`, + ); case 'openai-compatible': { // D12=A: unified auth via Recipe.resolveAuth (or default). const auth = applyResolveAuth(recipe, cfg, 'embedding'); @@ -2395,6 +2399,15 @@ function instantiateExpansion(recipe: Recipe, modelId: string, cfg: AIGatewayCon const baseURL = resolveNativeBaseUrl('anthropic', cfg); return createAnthropic({ apiKey, ...(baseURL ? { baseURL } : {}) }).languageModel(modelId); } + case 'claude-cli': { + // The CLI handles its own auth (OAuth session); spawn the subprocess + // directly via the same LanguageModelV2 implementation chat uses. There + // is no separate expansion path because claude-cli does not declare a + // separate expansion touchpoint — but routing here keeps the switch + // exhaustive and lets a future expansion touchpoint use the same code. + const { ClaudeCliLanguageModel } = require('./providers/claude-cli-language-model.ts'); + return new ClaudeCliLanguageModel(modelId); + } case 'openai-compatible': { // D12=A: unified auth via Recipe.resolveAuth (or default). const auth = applyResolveAuth(recipe, cfg, 'expansion'); @@ -2894,6 +2907,15 @@ function instantiateChat(recipe: Recipe, modelId: string, cfg: AIGatewayConfig): const baseURL = resolveNativeBaseUrl('anthropic', cfg); return createAnthropic({ apiKey, ...(baseURL ? { baseURL } : {}) }).languageModel(modelId); } + case 'claude-cli': { + // The CLI handles its own auth (OAuth session managed by `claude` + // login). Subprocess-based LanguageModelV2 dispatches via the recipe + // path so per-call routing works: `claude-cli:claude-sonnet-4-6` lands + // here, while sibling `litellm:gpt-5.4` continues through the + // openai-compatible path below. No env-var switch, no global flag. + const { ClaudeCliLanguageModel } = require('./providers/claude-cli-language-model.ts'); + return new ClaudeCliLanguageModel(modelId); + } case 'openai-compatible': { // D12=A: unified auth via Recipe.resolveAuth (or default). const auth = applyResolveAuth(recipe, cfg, 'chat'); diff --git a/src/core/ai/providers/claude-cli-language-model.ts b/src/core/ai/providers/claude-cli-language-model.ts new file mode 100644 index 000000000..4ffaec0af --- /dev/null +++ b/src/core/ai/providers/claude-cli-language-model.ts @@ -0,0 +1,444 @@ +/** + * ai-sdk LanguageModelV2 implementation that dispatches via the `claude --print` + * CLI subprocess. Used by the `claude-cli` recipe to route gateway.toolLoop / + * gateway.chat calls through Claude Code's OAuth session instead of the + * Anthropic SDK + ANTHROPIC_API_KEY. + * + * Per-call routing is the contract: the gateway resolves the model string + * to this recipe based on the `claude-cli:` prefix, instantiates one of + * these objects per modelId, and dispatches doGenerate. Sibling subagent + * jobs with `litellm:gpt-5.4` continue routing through litellm-proxy in + * the same worker; no env-var switch, no global state. + * + * Tool use is supported via system-prompt-instructed JSON emission: + * The recipe injects a fenced instruction block into the system prompt + * that teaches the model the `[{id,name,input}, ...]` + * emission format. The adapter parses those blocks back into ai-sdk + * `tool-call` content parts. Parallel tool calls (multiple entries in + * the JSON array) round-trip cleanly — this is the case that breaks + * on the codex-proxy / litellm GPT-5.x bridge today. + * + * Context isolation: + * The subprocess is spawned from a dedicated tmpdir so claude-cli's + * CLAUDE.md auto-discovery has no local files to find. `--system-prompt` + * replaces the default system prompt; `--disable-slash-commands` skips + * skill resolution. User-level ~/.claude/CLAUDE.md still loads because + * the only way to skip it is `--bare`, which forces ANTHROPIC_API_KEY + * auth and defeats the whole point of this provider. The ~42k cached + * tokens from user-level instructions are accepted as a cost-trivial + * trade-off on the subscription path. + * + * doStream is not yet implemented; the model declares no streaming. Callers + * (gateway.toolLoop primarily) use doGenerate. + */ +import { spawn } from 'node:child_process'; +import { mkdirSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import type { + LanguageModelV2, + LanguageModelV2CallOptions, + LanguageModelV2Content, + LanguageModelV2FunctionTool, + LanguageModelV2Prompt, + LanguageModelV2Message, + LanguageModelV2ProviderDefinedTool, +} from '@ai-sdk/provider'; + +function claudeBin(): string { + return process.env.GBRAIN_CLAUDE_CLI_BIN ?? 'claude'; +} +const CLAUDE_CWD = join(tmpdir(), `gbrain-claude-cli-cwd-${process.pid}`); +let cwdEnsured = false; +function ensureCleanCwd(): string { + if (!cwdEnsured) { + mkdirSync(CLAUDE_CWD, { recursive: true }); + cwdEnsured = true; + } + return CLAUDE_CWD; +} + +/** Parsed shape of `claude --print --output-format json`. */ +interface ClaudeJsonResult { + type: 'result'; + subtype: 'success' | string; + is_error: boolean; + result: string; + stop_reason: string | null; + session_id: string; + num_turns: number; + usage?: { + input_tokens?: number; + output_tokens?: number; + cache_read_input_tokens?: number; + cache_creation_input_tokens?: number; + }; +} + +/** + * Build the system-prompt addendum that teaches the model the + * `...` emission format. Returns the empty string + * when no tools are registered for this turn so the model gets a normal + * text-completion prompt without protocol noise. + */ +function buildToolUseInstructions( + tools: ReadonlyArray | undefined, +): string { + if (!tools || tools.length === 0) return ''; + + const functionTools = tools.filter((t): t is LanguageModelV2FunctionTool => t.type === 'function'); + if (functionTools.length === 0) return ''; + + const toolSpecs = functionTools.map(t => ({ + name: t.name, + description: t.description ?? '', + input_schema: t.inputSchema ?? { type: 'object', properties: {} }, + })); + + return [ + '', + '## Tool Use Protocol', + '', + 'You have access to these tools:', + '', + '```json', + JSON.stringify(toolSpecs, null, 2), + '```', + '', + 'To call one or more tools in this turn, emit EXACTLY ONE block of this form, ' + + 'with no other text outside the block on its own lines:', + '', + '', + '[', + ' {"id": "", "name": "", "input": }', + ']', + '', + '', + 'Multiple tool calls go in the array. Tool results are returned to you on the ' + + 'next turn as [tool_result ] entries. You may then call more tools or emit a final response.', + '', + 'When you are ready to give a final answer instead of calling tools, respond with prose text only — ' + + 'do not include a block in that case.', + '', + ].join('\n'); +} + +/** + * Render the ai-sdk message array into a single text prompt for `claude --print` + * stdin. System messages are extracted up-front and concatenated into the + * `--system-prompt` flag value. Tool calls and tool results are rendered as + * placeholders so the model sees the conversation in a coherent shape even + * though the adapter does not natively round-trip tool calls through claude-cli. + */ +function renderPrompt(prompt: LanguageModelV2Prompt): { systemText: string; userPrompt: string } { + const systemParts: string[] = []; + const convo: string[] = []; + + for (const msg of prompt as ReadonlyArray) { + if (msg.role === 'system') { + systemParts.push(msg.content); + continue; + } + if (msg.role === 'user') { + const text = msg.content + .map(p => { + if (p.type === 'text') return p.text; + // File parts get a stub — multimodal is not supported via subprocess yet. + if (p.type === 'file') return `[file ${p.mediaType ?? 'unknown'}]`; + return ''; + }) + .filter(s => s.length > 0) + .join('\n'); + if (text) convo.push(`User: ${text}`); + continue; + } + if (msg.role === 'assistant') { + const rendered = msg.content + .map(p => { + if (p.type === 'text') return p.text; + if (p.type === 'reasoning') return ''; // dropped on replay + if (p.type === 'tool-call') { + return `[tool_use ${p.toolName}(${p.input})]`; + } + if (p.type === 'tool-result') { + const out = typeof p.output === 'string' ? p.output : JSON.stringify(p.output); + return `[tool_result ${out}]`; + } + return ''; + }) + .filter(s => s.length > 0) + .join('\n'); + if (rendered) convo.push(`Assistant: ${rendered}`); + continue; + } + if (msg.role === 'tool') { + const rendered = msg.content + .map(p => { + const out = typeof p.output === 'string' ? p.output : JSON.stringify(p.output); + return `[tool_result ${out}]`; + }) + .join('\n'); + if (rendered) convo.push(`User: ${rendered}`); + continue; + } + } + + return { systemText: systemParts.join('\n'), userPrompt: convo.join('\n\n') }; +} + +/** + * Spawn `claude --print` with the contamination-suppression flags and return + * the parsed `--output-format json` envelope. Aborts propagate to SIGTERM on + * the child. + */ +function runClaude( + systemPrompt: string, + userPrompt: string, + model: string, + signal?: AbortSignal, +): Promise { + return new Promise((resolve, reject) => { + const args = [ + '--print', + '--output-format', 'json', + '--model', model, + '--disable-slash-commands', + // Agent isolation: this subprocess must behave like a raw LLM, not a + // full Claude Code agent. `--tools ""` disables every built-in tool + // (Bash/Read/WebSearch/...); `--strict-mcp-config` ignores all user-level + // MCP servers (without it, each call would boot the user's MCP servers — + // including gbrain's own MCP → recursion + PGLite single-writer lock + // contention). Verified against claude CLI 2.1.145 --help. + '--tools', '', + '--strict-mcp-config', + ]; + if (systemPrompt) { + args.push('--system-prompt', systemPrompt); + } + // Env scrub: guarantee the CLI authenticates via its own OAuth session + // (subscription), never via an inherited API key. Without this, an + // ANTHROPIC_API_KEY in gbrain's env (the exact setup this recipe is meant + // to replace) silently flips billing to per-token API usage. + const env = { ...process.env }; + delete env.ANTHROPIC_API_KEY; + delete env.ANTHROPIC_AUTH_TOKEN; + delete env.ANTHROPIC_BASE_URL; + const child = spawn(claudeBin(), args, { + stdio: ['pipe', 'pipe', 'pipe'], + cwd: ensureCleanCwd(), + env, + }); + + let stdout = ''; + let stderr = ''; + child.stdout.on('data', chunk => { stdout += String(chunk); }); + child.stderr.on('data', chunk => { stderr += String(chunk); }); + + const onAbort = () => { + child.kill('SIGTERM'); + reject(new Error('claude-cli adapter aborted')); + }; + if (signal) { + if (signal.aborted) { + onAbort(); + return; + } + signal.addEventListener('abort', onAbort, { once: true }); + } + + child.on('error', err => { + if (signal) signal.removeEventListener('abort', onAbort); + reject(new Error(`claude-cli spawn failed: ${err instanceof Error ? err.message : String(err)}`)); + }); + + child.on('close', code => { + if (signal) signal.removeEventListener('abort', onAbort); + if (code !== 0) { + reject(new Error(`claude-cli exited ${code}: ${stderr.trim() || stdout.trim()}`)); + return; + } + try { + let parsed = JSON.parse(stdout) as unknown; + // Compat: when the user has `"verbose": true` in ~/.claude/settings.json, + // `--print --output-format json` emits an ARRAY of events + // ([{type:"system",subtype:"init",...}, ..., {type:"result",...}]) + // instead of the bare result object. There is no CLI flag to force it + // off (no --no-verbose; --settings '{}' merges, does not replace), so + // tolerate both shapes and pick the result event. Verified on CLI 2.1.145. + if (Array.isArray(parsed)) { + const resultEvent = parsed.find( + (ev): ev is ClaudeJsonResult => + !!ev && typeof ev === 'object' && (ev as { type?: unknown }).type === 'result', + ); + if (!resultEvent) { + reject(new Error(`claude-cli JSON event array had no "result" event\n--- raw ---\n${stdout.slice(0, 500)}`)); + return; + } + parsed = resultEvent; + } + const envelope = parsed as ClaudeJsonResult; + if (envelope.is_error) { + reject(new Error(`claude-cli reported error: ${envelope.result || envelope.subtype}`)); + return; + } + resolve(envelope); + } catch (e) { + reject(new Error(`claude-cli output not JSON: ${e instanceof Error ? e.message : String(e)}\n--- raw ---\n${stdout.slice(0, 500)}`)); + } + }); + + // stdin error handler: if the binary does not exist (ENOENT) or the child + // dies before draining stdin, write/end can emit an unhandled 'error' + // (EPIPE) that would crash the worker. The spawn-level 'error' / non-zero + // 'close' handlers above already surface the real failure, so the stdin + // error itself is safe to swallow. + child.stdin.on('error', () => { /* surfaced via child 'error'/'close' */ }); + try { + child.stdin.write(userPrompt); + child.stdin.end(); + } catch (e) { + if (signal) signal.removeEventListener('abort', onAbort); + reject(new Error(`claude-cli stdin write failed (is the claude binary installed?): ${e instanceof Error ? e.message : String(e)}`)); + } + }); +} + +interface ParsedToolCall { + id: string; + name: string; + /** Stringified JSON, matching the ai-sdk LanguageModelV2ToolCall.input contract. */ + input: string; +} + +/** + * Locate and parse the `...` block in the assistant's + * raw text response. Returns the parsed tool calls plus whatever prose + * surrounded the block. Returns an empty `toolCalls` array when no block is + * present, malformed, or unterminated — the caller then treats the full + * raw text as a final text response. + */ +function extractToolCalls(raw: string): { + toolCalls: ParsedToolCall[]; + beforeText: string; + afterText: string; +} { + const openTag = ''; + const closeTag = ''; + const openIdx = raw.indexOf(openTag); + if (openIdx === -1) { + return { toolCalls: [], beforeText: raw.trim(), afterText: '' }; + } + const closeIdx = raw.indexOf(closeTag, openIdx + openTag.length); + if (closeIdx === -1) { + // Unterminated block — recover gracefully. + return { toolCalls: [], beforeText: raw.trim(), afterText: '' }; + } + + const beforeText = raw.slice(0, openIdx).trim(); + const afterText = raw.slice(closeIdx + closeTag.length).trim(); + let inner = raw.slice(openIdx + openTag.length, closeIdx).trim(); + + if (inner.startsWith('```')) { + inner = inner.replace(/^```(?:json|JSON)?\s*\n?/, '').replace(/\n?```$/, '').trim(); + } + + let parsed: unknown; + try { + parsed = JSON.parse(inner); + } catch { + return { toolCalls: [], beforeText: raw.trim(), afterText: '' }; + } + if (!Array.isArray(parsed)) { + return { toolCalls: [], beforeText: raw.trim(), afterText: '' }; + } + + const toolCalls: ParsedToolCall[] = []; + for (const entry of parsed) { + if (!entry || typeof entry !== 'object') continue; + const e = entry as Record; + const name = typeof e.name === 'string' ? e.name : null; + if (!name) continue; + const id = typeof e.id === 'string' && e.id.length > 0 + ? e.id + : `toolu_claude_cli_${Math.random().toString(36).slice(2, 12)}`; + const inputJson = JSON.stringify(e.input ?? {}); + toolCalls.push({ id, name, input: inputJson }); + } + + return { toolCalls, beforeText, afterText }; +} + +/** + * Strip provider prefixes (`anthropic:`, `litellm:`, `claude-cli:`) that the + * underlying CLI does not understand. The gateway hands us a bare model id + * via `recipe.aliases` resolution, but defensive normalization here keeps + * direct LanguageModelV2 construction (in tests, for example) ergonomic. + */ +function normalizeModel(model: string): string { + const idx = model.indexOf(':'); + return idx >= 0 ? model.slice(idx + 1) : model; +} + +export class ClaudeCliLanguageModel implements LanguageModelV2 { + readonly specificationVersion = 'v2' as const; + readonly provider = 'claude-cli'; + readonly modelId: string; + readonly supportedUrls = {}; + + constructor(modelId: string) { + this.modelId = normalizeModel(modelId); + } + + async doGenerate(options: LanguageModelV2CallOptions): Promise<{ + content: LanguageModelV2Content[]; + finishReason: 'stop' | 'length' | 'content-filter' | 'tool-calls' | 'error' | 'other' | 'unknown'; + usage: { inputTokens: number | undefined; outputTokens: number | undefined; totalTokens: number | undefined }; + warnings: never[]; + }> { + const { systemText, userPrompt } = renderPrompt(options.prompt); + const toolInstructions = buildToolUseInstructions(options.tools); + const systemPrompt = [systemText, toolInstructions].filter(s => s.length > 0).join('\n'); + + const result = await runClaude(systemPrompt, userPrompt, this.modelId, options.abortSignal); + const { toolCalls, beforeText, afterText } = extractToolCalls(result.result); + + const content: LanguageModelV2Content[] = []; + if (beforeText) content.push({ type: 'text', text: beforeText }); + for (const call of toolCalls) { + content.push({ + type: 'tool-call', + toolCallId: call.id, + toolName: call.name, + input: call.input, + }); + } + if (afterText) content.push({ type: 'text', text: afterText }); + if (content.length === 0) { + // Empty response — still hand the caller a well-formed content array. + content.push({ type: 'text', text: result.result ?? '' }); + } + + const finishReason = toolCalls.length > 0 ? 'tool-calls' as const : 'stop' as const; + const inputTokens = result.usage?.input_tokens; + const outputTokens = result.usage?.output_tokens; + const totalTokens = (inputTokens ?? 0) + (outputTokens ?? 0); + + return { + content, + finishReason, + usage: { + inputTokens, + outputTokens, + totalTokens: inputTokens !== undefined && outputTokens !== undefined ? totalTokens : undefined, + }, + warnings: [], + }; + } + + async doStream(): Promise { + throw new Error( + 'claude-cli LanguageModel does not support streaming. Use doGenerate or set ' + + 'the model on a non-streaming chat surface (gateway.toolLoop is non-streaming).', + ); + } +} diff --git a/src/core/ai/recipes/claude-cli.ts b/src/core/ai/recipes/claude-cli.ts new file mode 100644 index 000000000..2f1accbfe --- /dev/null +++ b/src/core/ai/recipes/claude-cli.ts @@ -0,0 +1,71 @@ +import type { Recipe } from '../types.ts'; + +/** + * Claude via the local `claude` CLI binary, using its built-in OAuth session + * (Claude Code / Claude Max subscription). No ANTHROPIC_API_KEY needed — the + * CLI manages its own auth state and the gateway dispatches via subprocess. + * + * Solves the #334 case where Max subscribers want Minions subagent dispatch + * to run against their existing subscription instead of paying per-token API + * charges. The recipe sits alongside the existing `anthropic` recipe so users + * pick per call: `anthropic:claude-sonnet-4-6` (API key + per-token billing) + * vs `claude-cli:claude-sonnet-4-6` (OAuth subscription, no API key). + * + * Chat-only. Claude has no first-party embedding model; users wanting an + * Anthropic chat path with embeddings still combine this with openai/google/ + * voyage for embedding the way the existing `anthropic` recipe documents. + * + * Auth: `auth_env.required: []` because the CLI handles auth itself. The + * `claude` binary on PATH (or `GBRAIN_CLAUDE_CLI_BIN`) IS the auth surface; + * there is nothing for the gateway to forward. + * + * Setup expectation: `claude` CLI installed and logged in (Claude Code + * onboarding does this), or `GBRAIN_CLAUDE_CLI_BIN` pointing at the binary. + */ +export const claudeCli: Recipe = { + id: 'claude-cli', + name: 'Claude (via CLI)', + tier: 'native', + implementation: 'claude-cli', + // The CLI owns auth; no env vars are required from the gateway side. + auth_env: { + required: [], + }, + touchpoints: { + // No embedding or expansion touchpoints — chat-only. + chat: { + models: [ + 'claude-opus-4-7', + 'claude-sonnet-4-6', + 'claude-haiku-4-5-20251001', + ], + supports_tools: true, + supports_subagent_loop: true, + // The CLI handles caching internally and does not surface it via the + // standard cache_control control plane. From the gateway's POV the + // model does not support prompt caching. + supports_prompt_cache: false, + max_context_tokens: 200000, + // Cost figures match the underlying Claude API tier, but the actual + // bill is borne by the subscription. We report them for the budget + // ledger's per-call accounting; operators on flat-rate subscriptions + // can treat the numbers as nominal. + cost_per_1m_input_usd: 3.0, + cost_per_1m_output_usd: 15.0, + price_last_verified: '2026-06-17', + }, + }, + // Friendly aliases mirror the `anthropic` recipe so config strings stay + // portable: switching `anthropic:claude-sonnet-4-6` to `claude-cli:claude-sonnet-4-6` + // is a one-token edit. Reverse aliases rewrite legacy IDs back to canonical. + aliases: { + 'claude-haiku-4-5': 'claude-haiku-4-5-20251001', + 'claude-sonnet-4-6-20250929': 'claude-sonnet-4-6', + 'sonnet': 'claude-sonnet-4-6', + 'haiku': 'claude-haiku-4-5-20251001', + 'opus': 'claude-opus-4-7', + }, + setup_hint: + 'Install Claude Code (`claude` CLI) and run `claude` once to log in. ' + + 'Set GBRAIN_CLAUDE_CLI_BIN if the binary is not on PATH.', +}; diff --git a/src/core/ai/recipes/index.ts b/src/core/ai/recipes/index.ts index e9f99f97c..a91010291 100644 --- a/src/core/ai/recipes/index.ts +++ b/src/core/ai/recipes/index.ts @@ -9,6 +9,7 @@ import type { Recipe } from '../types.ts'; import { openai } from './openai.ts'; import { google } from './google.ts'; import { anthropic } from './anthropic.ts'; +import { claudeCli } from './claude-cli.ts'; import { ollama } from './ollama.ts'; import { openrouter } from './openrouter.ts'; import { voyage } from './voyage.ts'; @@ -32,6 +33,7 @@ const ALL: Recipe[] = [ openai, google, anthropic, + claudeCli, ollama, openrouter, voyage, diff --git a/src/core/ai/types.ts b/src/core/ai/types.ts index 6b994c1fe..c96c6db21 100644 --- a/src/core/ai/types.ts +++ b/src/core/ai/types.ts @@ -22,7 +22,8 @@ export type Implementation = | 'native-openai' | 'native-google' | 'native-anthropic' - | 'openai-compatible'; + | 'openai-compatible' + | 'claude-cli'; export interface EmbeddingTouchpoint { models: string[]; diff --git a/test/claude-cli-recipe.test.ts b/test/claude-cli-recipe.test.ts new file mode 100644 index 000000000..26339b457 --- /dev/null +++ b/test/claude-cli-recipe.test.ts @@ -0,0 +1,535 @@ +/** + * Tests for the claude-cli LanguageModelV2 implementation that the + * `claude-cli` recipe instantiates. + * + * Strategy: a POSIX shell stub at GBRAIN_CLAUDE_CLI_BIN emits scripted + * --output-format json envelopes. Tests exercise the LanguageModelV2 + * doGenerate surface: text round trip, tool-call extraction (single + + * multiple parallel), abort semantics, context-isolation flags. No + * claude-cli installation or API credits required. + * + * Recipe registration is also smoke-tested: getRecipe('claude-cli') + * returns a chat-only Recipe with the right model list. + * + * Env isolation: GBRAIN_CLAUDE_CLI_BIN is set per-test via withEnv(), + * NOT in beforeAll. The provider reads the env var at spawn time so + * withEnv's save/restore in try/finally is sufficient; no leakage to + * sibling test files in the same bun-test process. + */ +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { writeFileSync, chmodSync, mkdirSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import type { LanguageModelV2CallOptions } from '@ai-sdk/provider'; +import { withEnv } from './helpers/with-env.ts'; + +const stubDir = join(tmpdir(), `claude-cli-recipe-stub-${process.pid}`); +const stubBin = join(stubDir, 'claude'); +const stubResponsePath = join(stubDir, 'claude_response.json'); + +beforeAll(() => { + mkdirSync(stubDir, { recursive: true }); + const stub = [ + '#!/bin/sh', + 'cat > /dev/null', + 'case " $* " in', + ' *" --print "*) ;;', + ' *) echo "missing --print in argv: $*" >&2; exit 64 ;;', + 'esac', + `cat "${stubResponsePath}"`, + ].join('\n'); + writeFileSync(stubBin, stub); + chmodSync(stubBin, 0o755); +}); + +afterAll(() => { + rmSync(stubDir, { recursive: true, force: true }); +}); + +function withStubEnv(fn: () => T | Promise): Promise { + return withEnv({ GBRAIN_CLAUDE_CLI_BIN: stubBin }, fn); +} + +function stageResponse(envelope: Record): void { + writeFileSync(stubResponsePath, JSON.stringify(envelope)); +} + +function baseEnvelope(result: string, overrides: Record = {}): Record { + return { + type: 'result', + subtype: 'success', + is_error: false, + result, + stop_reason: 'end_turn', + session_id: 'test-session', + num_turns: 1, + usage: { + input_tokens: 12, + output_tokens: 34, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + ...overrides, + }; +} + +function userMessage(text: string): LanguageModelV2CallOptions['prompt'][number] { + return { role: 'user', content: [{ type: 'text', text }] }; +} + +describe('claude-cli recipe registration', () => { + test('getRecipe returns chat-only Recipe with the documented models', async () => { + const { getRecipe } = await import('../src/core/ai/recipes/index.ts'); + const recipe = getRecipe('claude-cli'); + expect(recipe).toBeDefined(); + expect(recipe!.id).toBe('claude-cli'); + expect(recipe!.implementation).toBe('claude-cli'); + expect(recipe!.touchpoints.chat).toBeDefined(); + expect(recipe!.touchpoints.chat!.supports_tools).toBe(true); + expect(recipe!.touchpoints.chat!.supports_subagent_loop).toBe(true); + expect(recipe!.touchpoints.chat!.models).toContain('claude-sonnet-4-6'); + expect(recipe!.touchpoints.embedding).toBeUndefined(); + expect(recipe!.touchpoints.expansion).toBeUndefined(); + }); + + test('recipe aliases map short names to canonical model ids', async () => { + const { getRecipe } = await import('../src/core/ai/recipes/index.ts'); + const recipe = getRecipe('claude-cli'); + expect(recipe!.aliases!['sonnet']).toBe('claude-sonnet-4-6'); + expect(recipe!.aliases!['haiku']).toBe('claude-haiku-4-5-20251001'); + }); +}); + +describe('claude-cli LanguageModel — text-only round trip', () => { + test('returns a single text content block with usage + stop finish reason', async () => { + await withStubEnv(async () => { + stageResponse(baseEnvelope('hello world')); + const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts'); + const model = new ClaudeCliLanguageModel('claude-sonnet-4-6'); + const result = await model.doGenerate({ + prompt: [userMessage('hi')], + } as LanguageModelV2CallOptions); + + expect(result.finishReason).toBe('stop'); + expect(result.content).toHaveLength(1); + expect(result.content[0]).toEqual({ type: 'text', text: 'hello world' }); + expect(result.usage.inputTokens).toBe(12); + expect(result.usage.outputTokens).toBe(34); + }); + }); + + test('strips provider prefixes from the model id', async () => { + const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts'); + const model = new ClaudeCliLanguageModel('anthropic:claude-sonnet-4-6'); + expect(model.modelId).toBe('claude-sonnet-4-6'); + }); +}); + +describe('claude-cli LanguageModel — tool use', () => { + test('parses block into LanguageModelV2 tool-call content', async () => { + await withStubEnv(async () => { + stageResponse( + baseEnvelope( + [ + 'I will look up the pattern first.', + '', + '[{"id": "toolu_01ABC", "name": "search", "input": {"query": "n+1 query"}}]', + '', + ].join('\n'), + ), + ); + const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts'); + const model = new ClaudeCliLanguageModel('claude-sonnet-4-6'); + const result = await model.doGenerate({ + prompt: [userMessage('find n+1 queries')], + tools: [ + { + type: 'function', + name: 'search', + description: 'Search the brain', + inputSchema: { type: 'object', properties: { query: { type: 'string' } } }, + }, + ], + } as LanguageModelV2CallOptions); + + expect(result.finishReason).toBe('tool-calls'); + expect(result.content).toHaveLength(2); + expect(result.content[0]).toMatchObject({ type: 'text', text: 'I will look up the pattern first.' }); + expect(result.content[1]).toMatchObject({ + type: 'tool-call', + toolCallId: 'toolu_01ABC', + toolName: 'search', + input: '{"query":"n+1 query"}', + }); + }); + }); + + test('parses multiple parallel tool calls in a single block', async () => { + await withStubEnv(async () => { + stageResponse( + baseEnvelope( + [ + '', + '[', + ' {"id": "toolu_A", "name": "search", "input": {"query": "foo"}},', + ' {"id": "toolu_B", "name": "get_page", "input": {"slug": "areas/x"}}', + ']', + '', + ].join('\n'), + ), + ); + const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts'); + const model = new ClaudeCliLanguageModel('claude-sonnet-4-6'); + const result = await model.doGenerate({ + prompt: [userMessage('multi')], + tools: [ + { type: 'function', name: 'search', description: 's', inputSchema: { type: 'object', properties: {} } }, + { type: 'function', name: 'get_page', description: 'g', inputSchema: { type: 'object', properties: {} } }, + ], + } as LanguageModelV2CallOptions); + + const calls = result.content.filter(c => c.type === 'tool-call'); + expect(calls).toHaveLength(2); + expect(calls.map(c => (c as { toolName: string }).toolName)).toEqual(['search', 'get_page']); + expect(result.finishReason).toBe('tool-calls'); + }); + }); + + test('tolerates fenced JSON inside ', async () => { + await withStubEnv(async () => { + stageResponse( + baseEnvelope( + [ + '', + '```json', + '[{"id": "toolu_F", "name": "search", "input": {"q": "x"}}]', + '```', + '', + ].join('\n'), + ), + ); + const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts'); + const model = new ClaudeCliLanguageModel('claude-sonnet-4-6'); + const result = await model.doGenerate({ + prompt: [userMessage('fenced')], + tools: [{ type: 'function', name: 'search', description: '', inputSchema: { type: 'object', properties: {} } }], + } as LanguageModelV2CallOptions); + + const calls = result.content.filter(c => c.type === 'tool-call'); + expect(calls).toHaveLength(1); + }); + }); + + test('synthesizes an id when the model omits it', async () => { + await withStubEnv(async () => { + stageResponse( + baseEnvelope( + [ + '', + '[{"name": "search", "input": {"q": "x"}}]', + '', + ].join('\n'), + ), + ); + const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts'); + const model = new ClaudeCliLanguageModel('claude-sonnet-4-6'); + const result = await model.doGenerate({ + prompt: [userMessage('no id')], + tools: [{ type: 'function', name: 'search', description: '', inputSchema: { type: 'object', properties: {} } }], + } as LanguageModelV2CallOptions); + + const call = result.content.find(c => c.type === 'tool-call') as { toolCallId: string } | undefined; + expect(call).toBeDefined(); + expect(call!.toolCallId).toMatch(/^toolu_claude_cli_/); + }); + }); + + test('falls back to text on malformed JSON', async () => { + await withStubEnv(async () => { + stageResponse( + baseEnvelope( + [ + '', + 'not valid json', + '', + ].join('\n'), + ), + ); + const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts'); + const model = new ClaudeCliLanguageModel('claude-sonnet-4-6'); + const result = await model.doGenerate({ + prompt: [userMessage('malformed')], + tools: [{ type: 'function', name: 'search', description: '', inputSchema: { type: 'object', properties: {} } }], + } as LanguageModelV2CallOptions); + + expect(result.content.filter(c => c.type === 'tool-call')).toHaveLength(0); + expect(result.finishReason).toBe('stop'); + }); + }); + + test('returns text-only stop when tools are offered but model declines to call any', async () => { + // Real-world case: the model decides the user's request does not require + // a tool call, ignores the use_tools protocol, and answers directly. + // The recipe still must return clean LanguageModelV2 output so the + // caller (gateway.toolLoop) can treat the text as the final answer + // rather than wedge waiting for tool calls that never come. + await withStubEnv(async () => { + stageResponse( + baseEnvelope( + 'I do not actually need to call any tools for this. The answer is 42.', + { stop_reason: 'end_turn' }, + ), + ); + const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts'); + const model = new ClaudeCliLanguageModel('claude-sonnet-4-6'); + const result = await model.doGenerate({ + prompt: [userMessage('what is the meaning of life? you may use tools but do not need to')], + tools: [{ type: 'function', name: 'compute', description: 'Compute things', inputSchema: { type: 'object', properties: {} } }], + } as LanguageModelV2CallOptions); + + // No tool-call content blocks; caller treats this as a final answer. + expect(result.content.filter(c => c.type === 'tool-call')).toHaveLength(0); + // Text block present with the full model reply. + const textBlocks = result.content.filter(c => c.type === 'text'); + expect(textBlocks).toHaveLength(1); + expect((textBlocks[0] as { text: string }).text).toContain('42'); + // finishReason 'stop' tells the gateway-loop this is terminal output, + // not a partial mid-tool-loop state. + expect(result.finishReason).toBe('stop'); + }); + }); + + test('drops the block when the close tag is missing', async () => { + await withStubEnv(async () => { + stageResponse( + baseEnvelope( + [ + '', + '[{"id": "toolu_X", "name": "search", "input": {}}', + ].join('\n'), + ), + ); + const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts'); + const model = new ClaudeCliLanguageModel('claude-sonnet-4-6'); + const result = await model.doGenerate({ + prompt: [userMessage('unterminated')], + tools: [{ type: 'function', name: 'search', description: '', inputSchema: { type: 'object', properties: {} } }], + } as LanguageModelV2CallOptions); + + expect(result.content.filter(c => c.type === 'tool-call')).toHaveLength(0); + expect(result.finishReason).toBe('stop'); + }); + }); +}); + +describe('claude-cli LanguageModel — context isolation', () => { + test('argv includes --disable-slash-commands + --system-prompt and cwd is the dedicated tmpdir', async () => { + await withStubEnv(async () => { + const argvLog = join(stubDir, 'argv.log'); + const cwdLog = join(stubDir, 'cwd.log'); + const recordStub = [ + '#!/bin/sh', + `printf "%s\\n" "$@" > "${argvLog}"`, + `pwd > "${cwdLog}"`, + 'cat > /dev/null', + `cat "${stubResponsePath}"`, + ].join('\n'); + writeFileSync(stubBin, recordStub); + chmodSync(stubBin, 0o755); + stageResponse(baseEnvelope('ok')); + + const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts'); + const model = new ClaudeCliLanguageModel('claude-sonnet-4-6'); + await model.doGenerate({ + prompt: [ + { role: 'system', content: 'You are gbrain subagent.' }, + userMessage('hi'), + ], + } as LanguageModelV2CallOptions); + + const fs = require('node:fs'); + const argv = fs.readFileSync(argvLog, 'utf8').split('\n').filter(Boolean); + const cwd = fs.readFileSync(cwdLog, 'utf8').trim(); + + expect(argv).toContain('--print'); + expect(argv).toContain('--output-format'); + expect(argv).toContain('json'); + expect(argv).toContain('--disable-slash-commands'); + // Agent-isolation hardening: no built-in tools, no inherited MCP servers. + expect(argv).toContain('--tools'); + expect(argv).toContain('--strict-mcp-config'); + expect(argv).toContain('--system-prompt'); + expect(argv).toContain('You are gbrain subagent.'); + expect(cwd).toMatch(/gbrain-claude-cli-cwd-\d+$/); + + const fastStub = [ + '#!/bin/sh', + 'cat > /dev/null', + `cat "${stubResponsePath}"`, + ].join('\n'); + writeFileSync(stubBin, fastStub); + chmodSync(stubBin, 0o755); + }); + }); + + test('scrubs ANTHROPIC_* credentials from the child env (subscription-only auth)', async () => { + await withStubEnv(async () => { + await withEnv( + { + ANTHROPIC_API_KEY: 'sk-should-never-leak', + ANTHROPIC_AUTH_TOKEN: 'tok-should-never-leak', + ANTHROPIC_BASE_URL: 'https://proxy.should.never.leak', + }, + async () => { + const envLog = join(stubDir, 'env.log'); + const envStub = [ + '#!/bin/sh', + `printf "key=%s\\ntoken=%s\\nbase=%s\\n" "\${ANTHROPIC_API_KEY:-UNSET}" "\${ANTHROPIC_AUTH_TOKEN:-UNSET}" "\${ANTHROPIC_BASE_URL:-UNSET}" > "${envLog}"`, + 'cat > /dev/null', + `cat "${stubResponsePath}"`, + ].join('\n'); + writeFileSync(stubBin, envStub); + chmodSync(stubBin, 0o755); + stageResponse(baseEnvelope('ok')); + + try { + const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts'); + const model = new ClaudeCliLanguageModel('claude-sonnet-4-6'); + await model.doGenerate({ + prompt: [userMessage('hi')], + } as LanguageModelV2CallOptions); + + const fs = require('node:fs'); + const seen = fs.readFileSync(envLog, 'utf8'); + expect(seen).toContain('key=UNSET'); + expect(seen).toContain('token=UNSET'); + expect(seen).toContain('base=UNSET'); + } finally { + const fastStub = [ + '#!/bin/sh', + 'cat > /dev/null', + `cat "${stubResponsePath}"`, + ].join('\n'); + writeFileSync(stubBin, fastStub); + chmodSync(stubBin, 0o755); + } + }, + ); + }); + }); +}); + +describe('claude-cli LanguageModel — abort + error envelopes', () => { + test('SIGTERMs the child on AbortSignal', async () => { + await withStubEnv(async () => { + const slowStub = [ + '#!/bin/sh', + 'cat > /dev/null', + 'sleep 30', + 'echo "{}"', + ].join('\n'); + writeFileSync(stubBin, slowStub); + chmodSync(stubBin, 0o755); + try { + const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts'); + const model = new ClaudeCliLanguageModel('claude-sonnet-4-6'); + const ac = new AbortController(); + const promise = model.doGenerate({ + prompt: [userMessage('slow')], + abortSignal: ac.signal, + } as LanguageModelV2CallOptions); + setTimeout(() => ac.abort(), 30); + await expect(promise).rejects.toThrow(/aborted/); + } finally { + const fastStub = [ + '#!/bin/sh', + 'cat > /dev/null', + `cat "${stubResponsePath}"`, + ].join('\n'); + writeFileSync(stubBin, fastStub); + chmodSync(stubBin, 0o755); + } + }); + }); + + test('rejects when stub reports is_error: true', async () => { + await withStubEnv(async () => { + stageResponse({ ...baseEnvelope('boom'), is_error: true }); + const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts'); + const model = new ClaudeCliLanguageModel('claude-sonnet-4-6'); + await expect( + model.doGenerate({ prompt: [userMessage('x')] } as LanguageModelV2CallOptions), + ).rejects.toThrow(/claude-cli reported error/); + }); + }); + + test('rejects on non-JSON output', async () => { + await withStubEnv(async () => { + writeFileSync(stubResponsePath, 'this is not json'); + const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts'); + const model = new ClaudeCliLanguageModel('claude-sonnet-4-6'); + await expect( + model.doGenerate({ prompt: [userMessage('x')] } as LanguageModelV2CallOptions), + ).rejects.toThrow(/claude-cli output not JSON/); + }); + }); + + test('accepts a verbose-mode JSON event array and picks the result event', async () => { + // With `"verbose": true` in ~/.claude/settings.json the CLI emits an array + // of events instead of the bare result object (no CLI flag disables it). + await withStubEnv(async () => { + writeFileSync( + stubResponsePath, + JSON.stringify([ + { type: 'system', subtype: 'init', session_id: 'test-session', tools: [], mcp_servers: [] }, + baseEnvelope('hello from array'), + ]), + ); + const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts'); + const model = new ClaudeCliLanguageModel('claude-sonnet-4-6'); + const result = await model.doGenerate({ + prompt: [userMessage('hi')], + } as LanguageModelV2CallOptions); + expect(result.finishReason).toBe('stop'); + expect(result.content[0]).toEqual({ type: 'text', text: 'hello from array' }); + }); + }); + + test('rejects a verbose-mode event array that lacks a result event', async () => { + // Verbose mode emits an event array; a truncated stream (or one carrying + // only init/system events) has no result event to unwrap. + await withStubEnv(async () => { + writeFileSync( + stubResponsePath, + JSON.stringify([ + { type: 'system', subtype: 'init', session_id: 'test-session', tools: [], mcp_servers: [] }, + { type: 'assistant', message: { role: 'assistant', content: [] } }, + ]), + ); + const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts'); + const model = new ClaudeCliLanguageModel('claude-sonnet-4-6'); + await expect( + model.doGenerate({ prompt: [userMessage('x')] } as LanguageModelV2CallOptions), + ).rejects.toThrow(/had no "result" event/); + }); + }); + + test('rejects cleanly when the claude binary is missing (no worker crash)', async () => { + // A missing binary must surface as a rejected promise via the spawn 'error' + // handler; the child stdin 'error' (EPIPE) handler swallows the pipe failure + // so it never escalates to an unhandled rejection that would down the worker. + await withEnv({ GBRAIN_CLAUDE_CLI_BIN: join(stubDir, 'nonexistent-claude') }, async () => { + const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts'); + const model = new ClaudeCliLanguageModel('claude-sonnet-4-6'); + await expect( + model.doGenerate({ prompt: [userMessage('x')] } as LanguageModelV2CallOptions), + ).rejects.toThrow(/claude-cli spawn failed/); + }); + }); + + test('doStream throws not-supported', async () => { + const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts'); + const model = new ClaudeCliLanguageModel('claude-sonnet-4-6'); + await expect(model.doStream()).rejects.toThrow(/does not support streaming/); + }); +}); From 2944d9b7ae01fa7960bcdd329ef765c517bbda3c Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:01:01 -0700 Subject: [PATCH 22/39] fix(dims): handle prefixed model IDs on openai-compatible path (#2325) (#3309) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OpenRouter (and potentially other proxy providers) expose OpenAI's text-embedding-3 models with a provider prefix in the model ID, e.g. `openai/text-embedding-3-large` rather than bare `text-embedding-3-large`. `dimsProviderOptions()` checks `modelId.startsWith('text-embedding-3')` which fails for the prefixed form, so the `dimensions` parameter is never sent. The upstream provider returns its native dimensionality (3072 for -large) instead of the configured value (e.g. 1536), causing an immediate "dim mismatch" error on first embed. The default OpenRouter embedding (`text-embedding-3-small` at 1536d) masked this because its native size happens to match the default config. The bug surfaces when using `-large`, or `-small` with a non-1536 dim (512, 768, 1024 — all listed in the recipe's `dims_options`). Fix: strip the provider prefix before the `startsWith` check. The full prefixed ID is preserved in the error message for user clarity. Co-authored-by: Noetherly <280958447+noetherly@users.noreply.github.com> --- src/core/ai/dims.ts | 7 ++++--- test/ai/dims-openai.test.ts | 27 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/core/ai/dims.ts b/src/core/ai/dims.ts index 75bf1a0ac..3f3e81c3c 100644 --- a/src/core/ai/dims.ts +++ b/src/core/ai/dims.ts @@ -285,9 +285,10 @@ export function dimsProviderOptions( // configured for a smaller width (e.g. 1536) hard-fail at first embed. // Azure/OpenAI-compat embeddings are symmetric — inputType ignored. // v0.36.0.0 (D13): same range validation as native-openai path. - if (modelId.startsWith('text-embedding-3')) { - if (isOpenAITextEmbedding3Model(modelId) && !isValidOpenAITextEmbedding3Dim(modelId, dims)) { - const max = maxOpenAITextEmbedding3Dim(modelId)!; + const bareModelId = modelId.includes('/') ? modelId.split('/').pop()! : modelId; + if (bareModelId.startsWith('text-embedding-3')) { + if (isOpenAITextEmbedding3Model(bareModelId) && !isValidOpenAITextEmbedding3Dim(bareModelId, dims)) { + const max = maxOpenAITextEmbedding3Dim(bareModelId)!; throw new AIConfigError( `OpenAI model "${modelId}" supports embedding_dimensions in 1..${max}, got ${dims}.`, `Set \`embedding_dimensions\` to a value between 1 and ${max} ` + diff --git a/test/ai/dims-openai.test.ts b/test/ai/dims-openai.test.ts index c1359fdfc..2d05a4dcb 100644 --- a/test/ai/dims-openai.test.ts +++ b/test/ai/dims-openai.test.ts @@ -134,3 +134,30 @@ describe('dimsProviderOptions — OpenAI on openai-compatible adapter (Azure cas expect(JSON.stringify(opts)).not.toContain('input_type'); }); }); + +describe('dimsProviderOptions — prefixed model IDs (OpenRouter / proxy providers)', () => { + test('openai/text-embedding-3-large at 1536d returns dimensions=1536', () => { + const opts = dimsProviderOptions('openai-compatible', 'openai/text-embedding-3-large', 1536); + expect(opts).toEqual({ openaiCompatible: { dimensions: 1536 } }); + }); + + test('openai/text-embedding-3-small at 768d returns dimensions=768', () => { + const opts = dimsProviderOptions('openai-compatible', 'openai/text-embedding-3-small', 768); + expect(opts).toEqual({ openaiCompatible: { dimensions: 768 } }); + }); + + test('openai/text-embedding-3-large at 5000d throws AIConfigError', () => { + expect(() => dimsProviderOptions('openai-compatible', 'openai/text-embedding-3-large', 5000)) + .toThrow(AIConfigError); + }); + + test('error message preserves full prefixed model ID for clarity', () => { + try { + dimsProviderOptions('openai-compatible', 'openai/text-embedding-3-large', 5000); + throw new Error('should have thrown'); + } catch (err) { + expect(err).toBeInstanceOf(AIConfigError); + expect((err as Error).message).toContain('openai/text-embedding-3-large'); + } + }); +}); From 9b3f1c678695d32822f675e037822a5ab6f12641 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:01:07 -0700 Subject: [PATCH 23/39] fix dream orphan source scope (#2368) (#3344) Co-authored-by: Haoqian --- src/commands/jobs.ts | 1 + src/core/cycle.ts | 35 ++++++++++++++++------- test/autopilot-global-maintenance.test.ts | 17 +++++++++-- test/core/cycle.serial.test.ts | 32 ++++++++++++++++++++- 4 files changed, 71 insertions(+), 14 deletions(-) diff --git a/src/commands/jobs.ts b/src/commands/jobs.ts index 599a934ff..4f7c6a240 100644 --- a/src/commands/jobs.ts +++ b/src/commands/jobs.ts @@ -1885,6 +1885,7 @@ export async function registerBuiltinHandlers( signal: job.signal, deadlineAtMs: job.deadlineAtMs, // #2781: phases budget sub-work from remaining time phases, + forceGlobalOrphans: true, yieldBetweenPhases: async () => { await new Promise((r) => setImmediate(r)); }, }); diff --git a/src/core/cycle.ts b/src/core/cycle.ts index ce043006d..a779bf5aa 100644 --- a/src/core/cycle.ts +++ b/src/core/cycle.ts @@ -203,8 +203,10 @@ export const ALL_PHASES: CyclePhase[] = [ * - `source`: safe to parallelize per source. Sync reads/writes the * one source's rows; extract walks changed slugs. * - `global`: must serialize across the brain. Embed walks all stale - * chunks; orphans/purge sweep brain-wide; grade_takes + calibration - * aggregate across sources; resolve_symbol_edges walks every chunk. + * chunks; purge sweeps brain-wide; orphans can report a single + * resolved source but still belongs in the serialized global lane; + * grade_takes + calibration aggregate across sources; + * resolve_symbol_edges walks every chunk. * - `mixed`: per-phase decomposition needed before parallelizing. * Synthesize reads the brain-global transcripts dir but writes to * per-source slugs (via subagent allowlist). Patterns reads @@ -466,6 +468,12 @@ export interface CycleOpts { * loop bug (codex finding #3). */ synthBypassDreamGuard?: boolean; + /** + * Force the orphans phase to scan brain-wide even when `brainDir` resolves to + * a source. Used by autopilot global maintenance, whose phase set is + * intentionally brain-wide. + */ + forceGlobalOrphans?: boolean; /** * AbortSignal from the Minions worker (v0.22.1, #403). When aborted * (timeout, cancel, lock-loss), runCycle bails between phases and @@ -482,12 +490,14 @@ export interface CycleOpts { * + every existing caller). * * **Note for follow-up waves:** this only scopes the LOCK. Several - * cycle phases (`embed`, `orphans`, `purge`, `resolve_symbol_edges`, - * `grade_takes`, `calibration_profile`) still operate brain-wide - * regardless of sourceId — see the `PHASE_SCOPE` taxonomy. Per-source - * cycle locks let two cycles RUN, but the global-scoped phases - * inside each will still touch the same rows. Genuine per-source - * fan-out requires the deferred TODOs in the plan. + * cycle phases (`embed`, `purge`, `resolve_symbol_edges`, `grade_takes`, + * `calibration_profile`) still operate brain-wide regardless of sourceId + * — see the `PHASE_SCOPE` taxonomy. `orphans` uses the resolved source + * for its candidate set when one exists, but it remains in the serialized + * global lane for autopilot scheduling. Per-source cycle locks let two + * cycles RUN, but the global-scoped phases inside each will still touch + * the same rows. Genuine per-source fan-out requires the deferred TODOs + * in the plan. * * Validated via `assertValidSourceId` in `cycleLockIdFor` (defense-in-depth). */ @@ -1439,10 +1449,10 @@ async function runPhasePurge(engine: BrainEngine, dryRun: boolean): Promise { +async function runPhaseOrphans(engine: BrainEngine, sourceId?: string): Promise { try { const { findOrphans } = await import('../commands/orphans.ts'); - const result = await findOrphans(engine); + const result = await findOrphans(engine, sourceId !== undefined ? { sourceId } : {}); const count = result.total_orphans; // Orphans are a code-smell signal, not a fatal condition. The // original `count > 20` cutoff was tuned for small dev brains; on @@ -1461,8 +1471,10 @@ async function runPhaseOrphans(engine: BrainEngine): Promise { summary: `${count} orphan page(s) out of ${result.total_pages} total`, details: { total_orphans: count, + total_linkable: result.total_linkable, total_pages: result.total_pages, excluded: result.excluded, + ...(sourceId !== undefined ? { source_id: sourceId } : {}), }, }; } catch (e) { @@ -1526,6 +1538,7 @@ export async function runCycle( const cycleSourceId: string | undefined = engine ? (opts.sourceId ?? (await resolveSourceForDir(engine, brainDir))) : opts.sourceId; + const orphansSourceId = opts.forceGlobalOrphans ? undefined : cycleSourceId; const progress = createProgress(cliOptsToProgressOptions(getCliOptions())); @@ -2340,7 +2353,7 @@ export async function runCycle( }); } else { progress.start('cycle.orphans'); - const { result, duration_ms } = await timePhase(() => runPhaseOrphans(engine)); + const { result, duration_ms } = await timePhase(() => runPhaseOrphans(engine, orphansSourceId)); result.duration_ms = duration_ms; phaseResults.push(result); progress.finish(); diff --git a/test/autopilot-global-maintenance.test.ts b/test/autopilot-global-maintenance.test.ts index 0ec76f290..c5c500d88 100644 --- a/test/autopilot-global-maintenance.test.ts +++ b/test/autopilot-global-maintenance.test.ts @@ -11,6 +11,9 @@ */ import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { PGLiteEngine } from '../src/core/pglite-engine.ts'; import { resetPgliteState } from './helpers/reset-pglite.ts'; import { registerBuiltinHandlers } from '../src/commands/jobs.ts'; @@ -130,13 +133,23 @@ describe('autopilot-global-maintenance handler stamps last_global_at (PGLite)', test('runs global phases (no source_id) and stamps autopilot.last_global_at on success', async () => { expect(await engine.getConfig(LAST_GLOBAL_AT_KEY)).toBeNull(); + const repoPath = mkdtempSync(join(tmpdir(), 'gbrain-global-maintenance-')); + await engine.executeRaw( + `INSERT INTO sources (id, name, local_path) VALUES ($1, $2, $3)`, + ['repo-a', 'repo-a', repoPath], + ); const handlers = await captureHandlers(); const handler = handlers.get('autopilot-global-maintenance'); expect(handler).toBeTruthy(); - const result = await handler!({ data: { phases: ['orphans', 'embed'] }, signal: undefined }); + const result = await handler!({ + data: { phases: ['orphans', 'embed'], repoPath }, + signal: undefined, + }); // The cycle ran the requested global phases (DB-only on an empty brain). - expect(result.report.phases.some((p: any) => p.phase === 'orphans')).toBe(true); + const orphans = result.report.phases.find((p: any) => p.phase === 'orphans'); + expect(orphans).toBeTruthy(); + expect(orphans.details.source_id).toBeUndefined(); expect(['ok', 'clean', 'partial']).toContain(result.report.status); // Freshness stamped so the dispatch gate backs off. const stamped = await engine.getConfig(LAST_GLOBAL_AT_KEY); diff --git a/test/core/cycle.serial.test.ts b/test/core/cycle.serial.test.ts index 8676bbd67..98acf8664 100644 --- a/test/core/cycle.serial.test.ts +++ b/test/core/cycle.serial.test.ts @@ -21,6 +21,7 @@ let syncCalls: Array<{ dryRun: boolean | undefined; noPull: boolean | undefined; let extractCalls: Array<{ mode: string; dir: string; slugs: string[] | undefined }> = []; let embedCalls: Array<{ stale: boolean | undefined; dryRun: boolean | undefined }> = []; let orphansCalls: number = 0; +let orphansOpts: Array<{ sourceId?: string } | undefined> = []; // Mock lint mock.module('../../src/commands/lint.ts', () => ({ @@ -98,8 +99,9 @@ mock.module('../../src/commands/embed.ts', () => ({ // Mock orphans mock.module('../../src/commands/orphans.ts', () => ({ - findOrphans: async () => { + findOrphans: async (_engine: any, opts?: { sourceId?: string }) => { orphansCalls++; + orphansOpts.push(opts); return { orphans: [], total_orphans: 1, @@ -148,6 +150,7 @@ beforeEach(() => { extractCalls = []; embedCalls = []; orphansCalls = 0; + orphansOpts = []; }); // ─── dryRun propagation (regression guards) ──────────────────────── @@ -215,6 +218,11 @@ describe('runCycle — phase selection', () => { expect(orphansCalls).toBe(1); expect(syncCalls.length).toBe(0); }); + + test('--phase orphans preserves explicit source scope', async () => { + await runCycle(sharedEngine, { brainDir: '/tmp/brain', phases: ['orphans'], sourceId: 'source-a' }); + expect(orphansOpts.at(-1)).toEqual({ sourceId: 'source-a' }); + }); }); // ─── Lock-skip for non-DB-write phase selections ────────────────── @@ -502,6 +510,28 @@ describe('runCycle — sourceId resolution (regression #475)', () => { expect(syncCalls.at(-1)?.sourceId).toBe('default'); }); + test('seeded sources row → orphans phase receives matching sourceId', async () => { + await (sharedEngine as any).db.query( + `INSERT INTO sources (id, name, local_path) VALUES ($1, $2, $3)`, + ['alpha', 'alpha', '/tmp/brain-2349-alpha'], + ); + await runCycle(sharedEngine, { brainDir: '/tmp/brain-2349-alpha', phases: ['orphans'] }); + expect(orphansOpts.at(-1)).toEqual({ sourceId: 'alpha' }); + }); + + test('forceGlobalOrphans keeps orphans brain-wide even when brainDir maps to a source', async () => { + await (sharedEngine as any).db.query( + `INSERT INTO sources (id, name, local_path) VALUES ($1, $2, $3)`, + ['global-source', 'global-source', '/tmp/brain-2349-global'], + ); + await runCycle(sharedEngine, { + brainDir: '/tmp/brain-2349-global', + phases: ['embed', 'orphans', 'purge'], + forceGlobalOrphans: true, + }); + expect(orphansOpts.at(-1)).toEqual({}); + }); + test('no matching sources row → performSync receives sourceId=undefined', async () => { await runCycle(sharedEngine, { brainDir: '/tmp/brain-475-b' }); expect(syncCalls.at(-1)?.sourceId).toBeUndefined(); From ca04874c8fd6601d5c6f6338d5890aac9de945d8 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:02:03 -0700 Subject: [PATCH 24/39] fix: Bun+Windows write-through EEXIST, non-Anthropic --max-cost pricing, dream-page exclusion in enrich (#2407) (#3316) * fix(write-through): guard mkdir against EEXIST on Bun+Windows * fix(budget): resolve non-Anthropic model pricing via canonical table under --max-cost * fix(enrich): exclude dream-generated pages from thin candidates --------- Co-authored-by: nguyenchiviet <40517873+nguyenchiviet@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 --- src/core/budget/budget-tracker.ts | 7 +++++++ src/core/pglite-engine.ts | 3 +++ src/core/postgres-engine.ts | 7 +++++++ src/core/write-through.ts | 6 +++++- 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/core/budget/budget-tracker.ts b/src/core/budget/budget-tracker.ts index fd59d8213..b51de3cf4 100644 --- a/src/core/budget/budget-tracker.ts +++ b/src/core/budget/budget-tracker.ts @@ -32,6 +32,7 @@ import { mkdirSync, appendFileSync } from 'node:fs'; import { dirname } from 'node:path'; import { gbrainPath } from '../config.ts'; import { ANTHROPIC_PRICING, type ModelPricing } from '../anthropic-pricing.ts'; +import { canonicalLookup } from '../model-pricing.ts'; import { EMBEDDING_PRICING, lookupEmbeddingPrice } from '../embedding-pricing.ts'; import { splitProviderModelId } from '../model-id.ts'; import { isoWeekFilename, resolveAuditDir } from '../audit-week-file.ts'; @@ -213,6 +214,12 @@ function lookupPricing(modelId: string, kind: BudgetKind): ModelPricing | null { if (kind === 'rerank' && providerId && FREE_LOCAL_RERANK_PROVIDERS.has(providerId)) { return { input: 0, output: 0 }; } + // Fall back to the full canonical pricing table so non-Anthropic chat + // models with a known price (openai:*, google:*, deepseek:*) resolve under + // --max-cost instead of TX2 no_pricing hard-failing at $0. ANTHROPIC_PRICING + // above is only the bare-keyed Claude view. + const canon = canonicalLookup(modelId); + if (canon) return canon; return null; } diff --git a/src/core/pglite-engine.ts b/src/core/pglite-engine.ts index c29b82887..0af4abc57 100644 --- a/src/core/pglite-engine.ts +++ b/src/core/pglite-engine.ts @@ -6007,6 +6007,9 @@ export class PGLiteEngine implements BrainEngine { ); } + // Exclude dream/synthesize-generated pages (parity with postgres-engine). + where.push(`(p.frontmatter ->> 'dream_generated') IS DISTINCT FROM 'true'`); + const orderKey = ENRICH_ORDER_SQL[opts.order] ? opts.order : 'inbound-links'; const orderBy = ENRICH_ORDER_SQL[orderKey]; diff --git a/src/core/postgres-engine.ts b/src/core/postgres-engine.ts index 55dbf9aea..396c48737 100644 --- a/src/core/postgres-engine.ts +++ b/src/core/postgres-engine.ts @@ -6293,6 +6293,12 @@ export class PostgresEngine implements BrainEngine { )` : sql``; + // Exclude dream/synthesize-generated pages (reflections, originals, cycle + // logs carrying frontmatter dream_generated:true). enrich develops ENTITY + // stubs; running it on a generated essay/log creates circular self-citation + // and drops the H1. IS DISTINCT FROM 'true' keeps NULL/'false' rows. + const dreamCondition = sql`AND (p.frontmatter ->> 'dream_generated') IS DISTINCT FROM 'true'`; + // Whitelisted ORDER BY (no injection — enum maps to a literal fragment). const orderKey = ENRICH_ORDER_SQL[opts.order] ? opts.order : 'inbound-links'; const orderBy = sql.unsafe(ENRICH_ORDER_SQL[orderKey]); @@ -6316,6 +6322,7 @@ export class PostgresEngine implements BrainEngine { AND (char_length(p.compiled_truth) + char_length(COALESCE(p.timeline, ''))) < ${threshold} ${sourceCondition} ${recencyCondition} + ${dreamCondition} ORDER BY ${orderBy} LIMIT ${limit} `; diff --git a/src/core/write-through.ts b/src/core/write-through.ts index e8ff4e71e..463964890 100644 --- a/src/core/write-through.ts +++ b/src/core/write-through.ts @@ -174,7 +174,11 @@ export async function writePageThrough( } } - mkdirSync(dirname(filePath), { recursive: true }); + // On Bun + Windows, mkdirSync(dir, { recursive: true }) can still throw + // EEXIST when the directory already exists (POSIX no-ops it). That aborts + // the put_page / enrich / capture write-through whenever the prefix dir + // already exists, silently leaving the DB and the .md file plane out of sync. + if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); // Atomic write: unique temp sibling + rename. Unique name (pid + random) // so two concurrent saves to the same target can't clobber each other's From b3891fa7fc0bec3939c356d4f49b7a49de1969b5 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:15:15 -0700 Subject: [PATCH 25/39] fix(chunkers/code): tolerate tiktoken special tokens in estimateTokens (#2453) (#3315) Code legitimately contains tiktoken special-token strings (e.g. CLIP/GPT tokenizers embed the literal <|endoftext|>). The default encode() uses disallowed_special='all' and THROWS on those, crashing reindex-code on valid source files. Re-encode treating them as ordinary text (allowed=[], disallowed=[]); heuristic fallback if even that fails. A token COUNT needs no special-token semantics. Co-authored-by: Jim Tang Co-authored-by: Claude Opus 4.8 (1M context) --- src/core/chunkers/code.ts | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/core/chunkers/code.ts b/src/core/chunkers/code.ts index 5578a290a..8145e98ac 100644 --- a/src/core/chunkers/code.ts +++ b/src/core/chunkers/code.ts @@ -1235,7 +1235,25 @@ export function estimateTokens(text: string): number { tiktokenInitialized = true; } if (tiktokenEncoder) { - return tiktokenEncoder.encode(text).length; + try { + return tiktokenEncoder.encode(text).length; + } catch { + // Code legitimately contains tiktoken special-token strings (e.g. CLIP/GPT + // tokenizers embed the literal "<|endoftext|>"). The default encode() uses + // disallowed_special='all' and THROWS on those, crashing reindex-code on + // valid source files. For a token COUNT we don't need special-token + // semantics: re-encode treating them as ordinary text (never throws), + // heuristic only if even that fails. + try { + return ( + tiktokenEncoder as unknown as { + encode: (s: string, allowed: string[], disallowed: string[]) => Uint32Array; + } + ).encode(text, [], []).length; + } catch { + return Math.max(1, Math.ceil(text.length / 4)); + } + } } return Math.max(1, Math.ceil(text.length / 4)); } From 454c26ab56f787997778dd24867c2d48a5eccf99 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:15:20 -0700 Subject: [PATCH 26/39] fix(init): point soul-audit hint at the conversational skill, not a nonexistent CLI verb (#2486) (#3314) Co-authored-by: Sean Gearin --- src/commands/init.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/init.ts b/src/commands/init.ts index 3a2ff7e08..55e168ddf 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -1552,7 +1552,7 @@ export function reportModStatus(): void { console.log(' cd ~/.claude/skills/gstack && ./setup'); } console.log('Resolver: skills/RESOLVER.md'); - console.log('Soul audit: run `gbrain soul-audit` to customize agent identity'); + console.log('Soul audit: ask your agent to "run a soul audit" to customize its identity (see skills/soul-audit)'); // Retrieval Reflex (#1981): the deterministic pointer layer is ON by default // (no action needed). The policy skill is installed into the HOST repo on // request — we PRINT the command rather than silently mutating the host repo. From 48d83bd200369ba8e85e5e927fa0638a4d9dc85c Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:15:24 -0700 Subject: [PATCH 27/39] fix(cycle): extract_facts guard requires live backing page, not just non-NULL entity_slug (#2497) (#3321) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The empty-fence guard counted every `row_num IS NULL AND entity_slug IS NOT NULL` row as a pending v0_32_2 backfill, but the inline facts writer keeps producing rows of exactly that shape post-migration: when a resolved slug has no fenceable page (slugify-floor / stub-guard-blocked unprefixed slugs like `wingman`, `people-jane-doe`), backstop.ts falls through to a DB-only insert with row_num NULL. Those rows are structurally unfenceable — no page to fence onto, and the ledger-complete migration won't re-run — so they jammed the phase forever (~16/day observed) and the warning advised a no-op `apply-migrations --yes`. Discriminator: a row is a genuine backfill candidate only if its entity_slug resolves to a LIVE page in the same source (EXISTS in `pages` with deleted_at NULL) — mirroring the migration's Phase B, which only fences slugs that map to a writable page. Genuine pre-v0.32.2 rows (their entity page exists) still gate; inline-writer unfenceable rows no longer do. Warning text updated to name the "entity page present, not yet fenced" condition. Regression tests pin both sides: unfenceable rows (no page / soft-deleted page) do NOT gate and the phase converges; a legacy row WITH a backing page still gates. Fails pre-fix, passes post-fix. (#2484) Reland note: original merge (53c90869) was batch-reverted (4b6cf32c) — the guard-semantics change broke test/phantom-redirect.test.ts 'round 2 P1: legacy-row guard fires BEFORE phantom-redirect pass', which seeded a legacy row WITHOUT a backing page and expected the guard to fire. Under the new (intended) semantics such a row is structurally unfenceable and must NOT gate. Fixed by seeding a live backing page for the legacy row, preserving what the test pins (guard fires before the phantom-redirect pass). Co-authored-by: Javier Aldape Co-authored-by: Claude Fable 5 --- src/core/cycle/extract-facts.ts | 70 ++++++++++++++++------ test/extract-facts-phase.test.ts | 100 +++++++++++++++++++++++++++++++ test/phantom-redirect.test.ts | 4 ++ 3 files changed, 157 insertions(+), 17 deletions(-) diff --git a/src/core/cycle/extract-facts.ts b/src/core/cycle/extract-facts.ts index d805fdec4..04ee53cea 100644 --- a/src/core/cycle/extract-facts.ts +++ b/src/core/cycle/extract-facts.ts @@ -23,14 +23,24 @@ * page coordinate only; legacy NULL-source_markdown_slug rows survive * because deleteFactsForPage targets source_markdown_slug = slug only. * - * Empty-fence guard (Codex R2-#7): the phase refuses to do its - * destructive reconciliation pass when legacy rows (row_num IS NULL, - * entity_slug IS NOT NULL) still exist in the brain — they're the - * v0.31 hot-memory facts pending the v0_32_2 backfill. Status returns - * `warn` with a hint to run `gbrain apply-migrations --yes`. Without - * the guard, an interrupted upgrade where v0_32_2 hasn't run could - * leave the cycle silently misreporting "0 facts on people/alice" - * while legacy rows linger in the DB. + * Empty-fence guard (Codex R2-#7; #2484): the phase refuses to do its + * destructive reconciliation pass when genuinely-backfillable legacy + * rows still exist — `row_num IS NULL` (never fenced) AND `entity_slug` + * resolves to a live page in this source (so the v0_32_2 migration's + * Phase B could fence them). Status returns `warn` with a hint to run + * `gbrain apply-migrations --yes`. Without the guard, an interrupted + * upgrade where v0_32_2 hasn't run could leave the cycle silently + * misreporting "0 facts on people/alice" while legacy rows linger. + * + * The live-page requirement (#2484) is load-bearing: the inline facts + * writer keeps producing `row_num IS NULL, entity_slug IS NOT NULL` + * rows AFTER the migration completes, whenever a resolved slug has no + * fenceable page (slugify-floor / stub-guard-blocked unprefixed slugs). + * Those are structurally unfenceable — no page to fence onto, and the + * ledger-complete migration won't re-run — so they must NOT gate, or + * the phase jams forever (~16/day observed). Requiring a backing page + * keeps genuine pre-v0.32.2 rows (whose entity page exists) gating + * while excluding the inline-writer's permanent-unfenceable rows. */ import type { BrainEngine } from '../engine.ts'; @@ -163,22 +173,48 @@ export async function runExtractFacts( phantomsMorePending: false, }; - // ── Empty-fence guard (Codex R2-#7) ──────────────────────────── - // Pre-check: if any legacy fact rows exist (row_num NULL but - // entity_slug NOT NULL), refuse to run the destructive - // reconciliation pass. The v0_32_2 orchestrator must complete - // first. + // ── Empty-fence guard (Codex R2-#7; #2484) ───────────────────── + // Pre-check: if any genuinely-backfillable legacy fact rows exist, + // refuse to run the destructive reconciliation pass — the v0_32_2 + // orchestrator must fence them first. + // + // A row is a real backfill candidate only when `row_num IS NULL` + // (never fenced) AND its `entity_slug` resolves to a LIVE page in + // this source (the migration's Phase B only fences rows whose + // entity_slug maps to a writable page). #2484: the original + // predicate was just `row_num IS NULL AND entity_slug IS NOT NULL`, + // which ALSO matched structurally-unfenceable hot-memory rows the + // inline writer keeps producing post-migration: the legacy DB-only + // fallback (backstop.ts) writes `entity_slug` (a resolved slug, e.g. + // a slugify-floor or stub-guard-blocked unprefixed slug like + // `people-jane-doe`) with `row_num` NULL whenever the slug has no + // fenceable page. Those rows can never satisfy the migration's exit + // condition (no page to fence onto, and `apply-migrations` is a + // ledger-complete no-op for them), so they jammed the phase forever + // — ~16/day, mislabeled "v0.31 pending backfill." We now require a + // live backing page, which both genuine pre-v0.32.2 rows (their + // entity page exists) satisfy and inline-writer unfenceable rows do + // not. const legacy = await engine.executeRaw<{ n: string }>( - `SELECT COUNT(*) AS n FROM facts WHERE row_num IS NULL AND entity_slug IS NOT NULL`, + `SELECT COUNT(*) AS n + FROM facts f + WHERE f.row_num IS NULL + AND f.entity_slug IS NOT NULL + AND EXISTS ( + SELECT 1 FROM pages p + WHERE p.source_id = f.source_id + AND p.slug = f.entity_slug + AND p.deleted_at IS NULL + )`, ); const legacyCount = parseInt(legacy[0]?.n ?? '0', 10); result.legacyRowsPending = legacyCount; if (legacyCount > 0) { result.guardTriggered = true; result.warnings.push( - `extract_facts: ${legacyCount} legacy v0.31 fact rows pending fence backfill. ` + - `Run \`gbrain apply-migrations --yes\` to complete v0_32_2 before this phase ` + - `can safely reconcile fence → DB.`, + `extract_facts: ${legacyCount} legacy v0.31 fact rows (entity page present, not yet ` + + `fenced) pending fence backfill. Run \`gbrain apply-migrations --yes\` to complete ` + + `v0_32_2 before this phase can safely reconcile fence → DB.`, ); return result; } diff --git a/test/extract-facts-phase.test.ts b/test/extract-facts-phase.test.ts index 1fdd24ef5..5f046dfdf 100644 --- a/test/extract-facts-phase.test.ts +++ b/test/extract-facts-phase.test.ts @@ -351,6 +351,106 @@ describe('runExtractFacts — empty-fence guard (Codex R2-#7)', () => { expect(r.guardTriggered).toBe(false); expect(r.factsInserted).toBe(1); }); + + // ── #2484: structurally-unfenceable hot-memory rows ─────────── + // The inline facts writer (backstop.ts) keeps producing + // `row_num IS NULL, entity_slug IS NOT NULL` rows AFTER the v0_32_2 + // migration completes: when a resolved slug has no fenceable page + // (slugify-floor / stub-guard-blocked unprefixed slugs like + // `wingman` or `people-jane-doe`), it falls through to a DB-only + // insert with row_num NULL. The OLD guard predicate + // (`row_num IS NULL AND entity_slug IS NOT NULL`) matched these and + // jammed the phase forever (~16/day) — they can never be fenced (no + // page to fence onto; the ledger-complete migration won't re-run). + // The fix requires a LIVE backing page, so these rows no longer gate. + test('#2484: unfenceable inline-writer rows (entity_slug set, NO backing page) do NOT trigger the guard', async () => { + // Two unfenceable rows whose entity_slug has no page row at all. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + await (engine as any).db.query( + `INSERT INTO facts (source_id, entity_slug, fact, kind, visibility, notability, + valid_from, source, confidence) + VALUES + ('default', 'wingman', 'handoff note A', 'fact', 'private', 'medium', now(), 'mcp:extract_facts', 1.0), + ('default', 'people-jane-doe', 'handoff note B', 'fact', 'private', 'medium', now(), 'mcp:extract_facts', 1.0)`, + ); + + // A real page with a fence that SHOULD reconcile (proves the phase + // converges past the guard rather than early-returning). + await putPage('people/alice', FACT_FENCE( + `| 1 | real fenced fact | fact | 1.0 | world | high | 2026-01-01 | | s | |`, + )); + + const r = await runExtractFacts(engine, { slugs: ['people/alice'] }); + + // Guard must NOT trip — the unfenceable rows are permanent by + // construction, not a migration blocker. + expect(r.guardTriggered).toBe(false); + expect(r.legacyRowsPending).toBe(0); + // The phase ran its reconcile pass (did not early-return). + expect(r.factsInserted).toBe(1); + + // The unfenceable rows survive untouched (still row_num NULL). + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const survivors = await (engine as any).db.query( + `SELECT entity_slug FROM facts WHERE row_num IS NULL ORDER BY entity_slug`, + ); + expect(survivors.rows.map((x: { entity_slug: string }) => x.entity_slug)) + .toEqual(['people-jane-doe', 'wingman']); + }); + + test('#2484: a genuine legacy row WITH a backing page still triggers the guard (discriminator stays sharp)', async () => { + // Same shape as the unfenceable row above (row_num NULL, entity_slug + // set) — the ONLY difference is a live backing page exists, so the + // migration's Phase B could fence it. This MUST still gate. + await putPage('people/bob', FACT_FENCE( + `| 1 | fence fact | fact | 1.0 | world | high | 2026-01-01 | | s | |`, + )); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + await (engine as any).db.query( + `INSERT INTO facts (source_id, entity_slug, fact, kind, visibility, notability, + valid_from, source, confidence) + VALUES ('default', 'people/bob', 'genuine legacy claim', 'fact', 'private', 'medium', + now(), 'mcp:put_page', 1.0)`, + ); + + const r = await runExtractFacts(engine, { slugs: ['people/bob'] }); + + expect(r.guardTriggered).toBe(true); + expect(r.legacyRowsPending).toBe(1); + expect(r.factsInserted).toBe(0); + expect(r.factsDeleted).toBe(0); + expect(r.warnings.some(w => w.includes('apply-migrations'))).toBe(true); + }); + + test('#2484: a soft-deleted backing page makes its legacy row unfenceable (does NOT gate)', async () => { + // Page exists then gets soft-deleted (deleted_at set). The migration + // can't fence onto a deleted page, so the row must not gate. + await putPage('people/carol', FACT_FENCE( + `| 1 | live fact | fact | 1.0 | world | high | 2026-01-01 | | s | |`, + )); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + await (engine as any).db.query( + `INSERT INTO facts (source_id, entity_slug, fact, kind, visibility, notability, + valid_from, source, confidence) + VALUES ('default', 'people/carol', 'orphaned legacy claim', 'fact', 'private', 'medium', + now(), 'mcp:put_page', 1.0)`, + ); + // Soft-delete the page. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + await (engine as any).db.query( + `UPDATE pages SET deleted_at = now() WHERE slug = 'people/carol' AND source_id = 'default'`, + ); + + // Reconcile a DIFFERENT live page so the phase has work to do. + await putPage('people/dave', FACT_FENCE( + `| 1 | dave fact | fact | 1.0 | world | high | 2026-01-01 | | s | |`, + )); + + const r = await runExtractFacts(engine, { slugs: ['people/dave'] }); + expect(r.guardTriggered).toBe(false); + expect(r.legacyRowsPending).toBe(0); + expect(r.factsInserted).toBe(1); + }); }); describe('runExtractFacts — multi-source isolation', () => { diff --git a/test/phantom-redirect.test.ts b/test/phantom-redirect.test.ts index 6798bc13e..a5d65ff57 100644 --- a/test/phantom-redirect.test.ts +++ b/test/phantom-redirect.test.ts @@ -584,6 +584,10 @@ describe('runExtractFacts — phantom-redirect integration', () => { await withTempDirs(async ({ brainDir }) => { // Seed a legacy v0.31 fact row (row_num NULL, entity_slug NOT NULL). // `source` is NOT NULL in the schema; the v0.31 path always set it. + // #2484: the guard only gates on rows whose entity_slug resolves to a + // LIVE page (genuine backfill candidates), so seed the backing page too. + await putPage('people/legacy', '# legacy\n', { type: 'person' }); + writeMd(brainDir, 'people/legacy', '# legacy\n'); await engine.executeRaw( `INSERT INTO facts (source_id, entity_slug, fact, kind, valid_from, source) VALUES ('default', 'people/legacy', 'Legacy claim', 'fact', '2020-01-01'::date, 'legacy-import')`, From 9e283790387ec574667df841fb5f13f3c64f58a5 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:16:17 -0700 Subject: [PATCH 28/39] fix: handle reasoning tags in parseExtractorOutput (#2559) (#3318) Reasoning models (MiniMax-M3, DeepSeek-R1, etc.) return ... tags in the content field before the actual JSON output. This caused parseExtractorOutput to fail in two ways: 1. The fence regex /^\`\`\`(json)?...$/ requires the fence at text start; preceding it prevents matching, so the raw text (with trailing fences) hits JSON.parse and throws. 2. When think tags contain [ or { characters, indexOf finds them inside the reasoning block instead of the actual JSON array. Changes: - Strip ... tags before any parsing (covers all reasoning models) - Add JSON.parse fallback: truncate at last ] or } to handle trailing noise (leftover markdown fences after stripping) Tests: 28/28 pass (3 new cases for think tags + trailing noise). Co-authored-by: qaz8545355 <603191978@qq.com> Co-authored-by: qaz8545355 --- src/core/cycle/propose-takes.ts | 18 +++++++++++++++++- test/propose-takes.test.ts | 20 ++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/src/core/cycle/propose-takes.ts b/src/core/cycle/propose-takes.ts index e6117fc02..c0ad7268e 100644 --- a/src/core/cycle/propose-takes.ts +++ b/src/core/cycle/propose-takes.ts @@ -299,6 +299,8 @@ export async function defaultExtractor( export function parseExtractorOutput(raw: string): ProposedTake[] { if (!raw || raw.trim().length === 0) return []; let text = raw.trim(); + // Strip ... reasoning tags (MiniMax-M3, DeepSeek-R1, etc.). + text = text.replace(/[\s\S]*?<\/think>/g, '').trim(); // Strip markdown code fence wrapper. const fenced = text.match(/^```(?:json)?\s*\n?([\s\S]*?)\n?```$/); if (fenced) text = (fenced[1] ?? '').trim(); @@ -311,7 +313,21 @@ export function parseExtractorOutput(raw: string): ProposedTake[] { try { parsed = JSON.parse(text.slice(start)); } catch { - return []; + // Fallback: truncate at last ] or } to handle trailing noise (e.g. leftover + // markdown fences after stripping). Try array-closing first. + const sliced = text.slice(start); + const lastArr = sliced.lastIndexOf(']'); + const lastObj = sliced.lastIndexOf('}'); + const end = Math.max(lastArr, lastObj); + if (end > 0) { + try { + parsed = JSON.parse(sliced.slice(0, end + 1)); + } catch { + return []; + } + } else { + return []; + } } const arr = Array.isArray(parsed) ? parsed : [parsed]; const out: ProposedTake[] = []; diff --git a/test/propose-takes.test.ts b/test/propose-takes.test.ts index 17a2e1d47..00919c2d0 100644 --- a/test/propose-takes.test.ts +++ b/test/propose-takes.test.ts @@ -172,6 +172,26 @@ describe('parseExtractorOutput', () => { const out = parseExtractorOutput(raw); expect(out[0]!.domain).toBe('macro'); }); + + test('strips reasoning tags before parsing (MiniMax-M3, DeepSeek-R1)', () => { + const raw = 'Analyzing the prose... I see several claims.\n\n```json\n[{"claim_text":"X","kind":"take","holder":"brain","weight":0.5}]\n```'; + const out = parseExtractorOutput(raw); + expect(out).toHaveLength(1); + expect(out[0]!.claim_text).toBe('X'); + }); + + test('strips multiple blocks', () => { + const raw = 'First thought.\n...\nSecond thought.\n\n[{"claim_text":"Y","kind":"bet","holder":"brain","weight":0.7}]'; + const out = parseExtractorOutput(raw); + expect(out).toHaveLength(1); + }); + + test('handles trailing noise after JSON (leftover fences)', () => { + const raw = 'done\n```json\n[{"claim_text":"Z","kind":"take","holder":"brain","weight":0.6}]\n```\n'; + const out = parseExtractorOutput(raw); + expect(out).toHaveLength(1); + expect(out[0]!.claim_text).toBe('Z'); + }); }); // ─── contentHash ──────────────────────────────────────────────────── From 06001248efe3cc49cd454893a81b3188fb52a78a Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:16:21 -0700 Subject: [PATCH 29/39] =?UTF-8?q?fix(storage):=20Supabase=20signed=20URLs?= =?UTF-8?q?=20=E2=80=94=20prepend=20/storage/v1=20(#2565)=20(#3320)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SupabaseStorage.getSignedUrl built the download URL as `${projectUrl}${signedURL}`, but Supabase's sign API returns `signedURL` relative to the Storage API root (/object/sign//?token=...), so the generated link dropped /storage/v1 and returned 404. Now prepends `${projectUrl}/storage/v1`, tolerating an already-absolute URL or a value that already carries the prefix. `gbrain files signed-url` links resolve again. Co-authored-by: FloridaStyle Co-authored-by: Claude Opus 4.8 --- src/core/storage/supabase.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/core/storage/supabase.ts b/src/core/storage/supabase.ts index ace1fb7f8..ef82d4d75 100644 --- a/src/core/storage/supabase.ts +++ b/src/core/storage/supabase.ts @@ -195,7 +195,14 @@ export class SupabaseStorage implements StorageBackend { throw new Error(`Supabase signed URL failed: ${res.status} ${body}`); } const result = await res.json() as { signedURL: string }; - return `${this.projectUrl}${result.signedURL}`; + // Supabase returns `signedURL` relative to the Storage API root, e.g. + // "/object/sign//?token=...". Prepend projectUrl + "/storage/v1" + // (not just projectUrl) or the link 404s. Tolerate an already-absolute URL or a + // value that already carries the /storage/v1 prefix. + const signed = result.signedURL; + if (/^https?:\/\//.test(signed)) return signed; + if (signed.startsWith('/storage/v1')) return `${this.projectUrl}${signed}`; + return `${this.projectUrl}/storage/v1${signed.startsWith('/') ? '' : '/'}${signed}`; } async getUrl(path: string): Promise { From 7fdecd5c01bda3ce05740b9b75f201248b823797 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:17:14 -0700 Subject: [PATCH 30/39] fix(minions): default timeout for contextual reindex (#2611) (#3323) Co-authored-by: spiky02plateau <155588579+spiky02plateau@users.noreply.github.com> --- src/core/minions/handler-timeouts.ts | 5 +++++ test/minions.test.ts | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/src/core/minions/handler-timeouts.ts b/src/core/minions/handler-timeouts.ts index 334e1f231..8269add1e 100644 --- a/src/core/minions/handler-timeouts.ts +++ b/src/core/minions/handler-timeouts.ts @@ -24,6 +24,7 @@ */ const THIRTY_MIN_MS = 30 * 60 * 1000; +const SIXTY_MIN_MS = 60 * 60 * 1000; const TEN_MIN_MS = 10 * 60 * 1000; /** @@ -42,6 +43,10 @@ export const HANDLER_DEFAULT_TIMEOUT_MS: Readonly> = { // few writes. Generous 10-min budget (vs the tight null-default) covers a // slow gateway without the 30-min loop budget. chronicle_extract: TEN_MIN_MS, + // Per-page contextual reindex jobs process chunks sequentially with one + // rate-leased LLM synopsis call per chunk; large transcript pages need more + // than the standard 30-min long-job budget. + contextual_reindex_per_chunk: SIXTY_MIN_MS, }; /** diff --git a/test/minions.test.ts b/test/minions.test.ts index 0909d7e44..90148361e 100644 --- a/test/minions.test.ts +++ b/test/minions.test.ts @@ -354,6 +354,13 @@ describe('MinionQueue: #1737 per-handler default timeout', () => { expect(sub.timeout_ms).toBe(30 * 60 * 1000); }); + test('contextual per-chunk reindex gets the 60-min default', async () => { + const job = await queue.add('contextual_reindex_per_chunk', { page_slug: 'large-transcript' }, undefined, { + allowProtectedSubmit: true, + }); + expect(job.timeout_ms).toBe(60 * 60 * 1000); + }); + test('explicit timeout_ms always wins over the default', async () => { const job = await queue.add('embed-backfill', { sourceId: 'x' }, { timeout_ms: 5000 }); expect(job.timeout_ms).toBe(5000); From 96465d8c3596687227eb39dc0b291218fedaec58 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:34:59 -0700 Subject: [PATCH 31/39] fix(migrations): let force-retry escape completed ledger entries (#2616) (#3325) statusForVersion short-circuited on any 'complete' entry before checking the trailing 'retry' marker, so --force-retry appended an inert row and a version marked complete with zero work done could never be re-run without hand-editing completed.jsonl. Check retry-latest first: an explicit --force-retry now yields 'pending' even past an earlier 'complete', while a stray 'partial' after 'complete' still cannot regress the version. Co-authored-by: spiky02plateau <155588579+spiky02plateau@users.noreply.github.com> --- src/commands/apply-migrations.ts | 13 ++++++------ test/apply-migrations.test.ts | 35 ++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/src/commands/apply-migrations.ts b/src/commands/apply-migrations.ts index c21d33cc4..9cd788a77 100644 --- a/src/commands/apply-migrations.ts +++ b/src/commands/apply-migrations.ts @@ -133,14 +133,15 @@ function indexCompleted(entries: CompletedMigrationEntry[]): CompletedIndex { * Returns the resolved status for a migration based on its entries. * * Semantics (Bug 3 — keep "complete wins" safety): - * - If any entry is `complete`, the version is complete. Terminal state. - * - Otherwise, if the latest entry is `retry`, the version is pending - * (user requested a fresh attempt). + * - If the latest entry is `retry`, the version is pending. This is the + * explicit escape hatch written by `--force-retry`, and it overrides an + * earlier `complete` entry without hand-editing the ledger. + * - Otherwise, if any entry is `complete`, the version is complete. * - Otherwise, if any entry is `partial`, the version is partial. * - Otherwise, pending. * - * `complete` never regresses. A later accidental `partial` append cannot - * undo a completed migration. + * `complete` never regresses accidentally. A later `partial` append cannot + * undo a completed migration; only a trailing, explicit `retry` marker can. */ function statusForVersion( version: string, @@ -148,9 +149,9 @@ function statusForVersion( ): 'complete' | 'partial' | 'pending' | 'wedged' { const entries = idx.byVersion.get(version) ?? []; if (entries.length === 0) return 'pending'; - if (entries.some(e => e.status === 'complete')) return 'complete'; const latest = entries[entries.length - 1]; if (latest.status === 'retry') return 'pending'; + if (entries.some(e => e.status === 'complete')) return 'complete'; // Bug 3 attempt cap — count consecutive partials from the end (stopping // at any 'retry' or 'complete'). If we hit MAX_CONSECUTIVE_PARTIALS, // the migration is wedged and needs explicit --force-retry to try again. diff --git a/test/apply-migrations.test.ts b/test/apply-migrations.test.ts index 06cdbaf2f..20690beaf 100644 --- a/test/apply-migrations.test.ts +++ b/test/apply-migrations.test.ts @@ -167,6 +167,41 @@ describe('buildPlan — diff against completed + installed VERSION', () => { }); }); +describe('force-retry escape hatch', () => { + test("complete then retry-latest → pending and buildPlan lists the version as pending", () => { + const idx = indexCompleted([ + { version: '0.11.0', status: 'complete' }, + { version: '0.11.0', status: 'retry' }, + ]); + + expect(statusForVersion('0.11.0', idx)).toBe('pending'); + const plan = buildPlan(idx, '0.11.1', '0.11.0'); + expect(plan.pending.map(m => m.version)).toEqual(['0.11.0']); + expect(plan.applied).toEqual([]); + expect(plan.partial).toEqual([]); + expect(plan.wedged).toEqual([]); + }); + + test('complete then stray partial without retry → still complete', () => { + const idx = indexCompleted([ + { version: '0.11.0', status: 'complete' }, + { version: '0.11.0', status: 'partial' }, + ]); + + expect(statusForVersion('0.11.0', idx)).toBe('complete'); + }); + + test('retry followed by a newer complete → complete', () => { + const idx = indexCompleted([ + { version: '0.11.0', status: 'complete' }, + { version: '0.11.0', status: 'retry' }, + { version: '0.11.0', status: 'complete' }, + ]); + + expect(statusForVersion('0.11.0', idx)).toBe('complete'); + }); +}); + // v0.36.1.x (cherry-pick #1062): list, dry-run, and "all migrations up to // date" paths must exit 0 so shell scripts gating on the exit code work. // Pre-fix, these `return` statements left the CLI dispatcher's implicit From 900ee3c678c6d1371a17af106cd93373c3966fd7 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:35:05 -0700 Subject: [PATCH 32/39] perf(contextual-retrieval): bound per-chunk synopsis concurrency (#2628) (#3326) Replace the strictly sequential per-chunk synopsis loop with a bounded sliding worker pool (existing runSlidingPool helper). Results land in chunk order via index-addressed writes; code chunks still bypass the wrapper; embedding remains one page-level batch after all synopses. New knob GBRAIN_CONTEXTUAL_CHUNK_CONCURRENCY, default 4, clamped to [1,16]; 1 reproduces the prior sequential behavior exactly. Each chunk task still acquires/releases the global synopsis rate-lease, which remains the cross-worker governor; the lease id now travels from acquire to release instead of shared mutable state, and lease waits are abort-responsive. At 20-45s per synopsis call, a 120-chunk transcript page previously needed 60-90+ min wall time and routinely outlived job timeouts. Co-authored-by: spiky02plateau <155588579+spiky02plateau@users.noreply.github.com> --- src/core/contextual-retrieval-service.ts | 243 +++++++++---- .../handlers/contextual-reindex-per-chunk.ts | 41 ++- .../contextual-retrieval-service-pure.test.ts | 344 +++++++++++++++++- 3 files changed, 533 insertions(+), 95 deletions(-) diff --git a/src/core/contextual-retrieval-service.ts b/src/core/contextual-retrieval-service.ts index a65c8e074..ed3e19ebb 100644 --- a/src/core/contextual-retrieval-service.ts +++ b/src/core/contextual-retrieval-service.ts @@ -45,7 +45,6 @@ import { embedBatch } from './embedding.ts'; import { resolveContextualRetrievalMode } from './contextual-retrieval-resolver.ts'; import { buildContextualPrefix, - extractFirstTwoSentences, modeRequiresHaiku, modeRequiresWrapper, sanitizeTitle, @@ -57,10 +56,8 @@ import { SYNOPSIS_DOC_MAX_CHARS, type GeneratePerChunkSynopsisResult, } from './page-summary.ts'; -import { - logSynopsisFailure, - type SynopsisFailureKind, -} from './audit-synopsis.ts'; +import type { SynopsisFailureKind } from './audit-synopsis.ts'; +import { runSlidingPool } from './worker-pool.ts'; import type { BrainEngine } from './engine.ts'; import type { ChunkInput, CRMode, Page } from './types.ts'; import type { SourceRow } from './sources-ops.ts'; @@ -73,6 +70,24 @@ import type { SourceRow } from './sources-ops.ts'; * corpus_generation hash. */ export const TITLE_WRAPPER_VERSION = 1; +const DEFAULT_HAIKU_MODEL = 'anthropic:claude-haiku-4-5-20251001'; +export const DEFAULT_CONTEXTUAL_CHUNK_CONCURRENCY = 4; +export const MAX_CONTEXTUAL_CHUNK_CONCURRENCY = 16; + +export function resolveContextualChunkConcurrency( + env: Record = process.env, +): number { + const raw = env.GBRAIN_CONTEXTUAL_CHUNK_CONCURRENCY; + if (raw === undefined || raw.trim() === '') return DEFAULT_CONTEXTUAL_CHUNK_CONCURRENCY; + const n = Number(raw); + if (!Number.isFinite(n)) return DEFAULT_CONTEXTUAL_CHUNK_CONCURRENCY; + return clampContextualChunkConcurrency(n); +} + +function clampContextualChunkConcurrency(n: number): number { + if (!Number.isFinite(n)) return DEFAULT_CONTEXTUAL_CHUNK_CONCURRENCY; + return Math.max(1, Math.min(MAX_CONTEXTUAL_CHUNK_CONCURRENCY, Math.trunc(n))); +} /** * Embedding model placeholder. The actual model name lands here from @@ -208,12 +223,16 @@ export interface ReembedPageArgs { * src/core/minions/rate-leases.ts here; inline callers (import-file, * reindex command) pass undefined and rely on gateway-level retry. */ - acquireSynopsisLease?: () => Promise; - releaseSynopsisLease?: () => Promise; + acquireSynopsisLease?: () => Promise; + releaseSynopsisLease?: (lease?: unknown) => Promise; + /** + * Intra-page per-chunk synopsis concurrency. 1 preserves the legacy + * sequential loop exactly; higher values only parallelize Haiku synopsis + * calls. Embedding remains one batch after all synopses succeed. + */ + chunkConcurrency?: number; } -const DEFAULT_HAIKU_MODEL = 'anthropic:claude-haiku-4-5-20251001'; - /** * Re-embed one page through the active CR mode. Implements the D26 P0-2 * two-phase build pattern. @@ -432,82 +451,41 @@ async function tryBuildPhase1(opts: { } // per_chunk_synopsis path. Read source text via fallback chain, - // generate synopsis per chunk sequentially within this page (D10), + // generate synopsis per chunk through a bounded sliding pool, then // batch embed at the end (D27 P2-2). const sourceText = readSourceTextWithFallback(page, chunks); - const wrappedTexts: string[] = []; + const wrappedTexts: string[] = new Array(chunks.length); + const chunkConcurrency = clampContextualChunkConcurrency( + args.chunkConcurrency ?? resolveContextualChunkConcurrency(), + ); - for (let i = 0; i < chunks.length; i++) { - const c = chunks[i]; - - // Code chunks always bypass the wrapper (D20-T4) — pass through. - if (c.chunk_source === 'fenced_code') { - wrappedTexts.push(c.chunk_text); - continue; - } - - // Acquire rate-lease per chunk (D26 P0-3). Inline callers pass no - // hooks; only the Minion handler wires through rate-leases.ts. - if (args.acquireSynopsisLease) { - await args.acquireSynopsisLease(); - } - - let synopsisResult: GeneratePerChunkSynopsisResult; - try { - synopsisResult = await generatePerChunkSynopsis({ - documentText: sourceText, - chunkText: c.chunk_text, - pageTitle: page.title, - pageSlug: args.pageSlug, - sourceId: args.sourceId, - chunkIndex: c.chunk_index, - model: haikuModel, - abortSignal: args.abortSignal, + const poolResult = await runSlidingPool({ + items: chunks, + workers: chunkConcurrency, + signal: args.abortSignal, + onError: 'abort', + failureLabel: (c) => String(c.chunk_index), + onItem: async (c, i) => { + wrappedTexts[i] = await buildWrappedChunkText({ + chunk: c, + sourceText, + safeTitle, + page, + args, + haikuModel, }); - } finally { - if (args.releaseSynopsisLease) { - try { - await args.releaseSynopsisLease(); - } catch { - // Lease release failure shouldn't abort the page; surfacing it - // would race with the synopsis result. Audit-only. - } - } - } + }, + }); - if (synopsisResult.kind === 'success') { - const prefix = buildContextualPrefix(safeTitle, synopsisResult.synopsis); - wrappedTexts.push( - wrapChunkForEmbedding(c.chunk_text, prefix, c.chunk_source), - ); - continue; + if (poolResult.failures.length > 0) { + const failure = [...poolResult.failures].sort((a, b) => a.idx - b.idx)[0].error; + if (failure instanceof ChunkSynopsisPhase1Error) { + return failure.result; } - - // Failure classification per D27 P1-2: - // refusal | empty | malformed → page-level fall-back to title-only - // auth_failure → permanent (won't fix with retry) - // rate_limit | timeout | network | provider_5xx → transient - // source_missing → walked into fallback already; would be 'malformed' - // from generatePerChunkSynopsis if we ever propagated it here - if ( - synopsisResult.kind === 'refusal' || - synopsisResult.kind === 'empty' || - synopsisResult.kind === 'malformed' - ) { - return { kind: 'page_level_fallback_requested', cause: synopsisResult.kind }; - } - if (synopsisResult.kind === 'auth_failure') { - return { - kind: 'permanent', - cause: synopsisResult.kind, - detail: synopsisResult.detail ?? 'auth failure', - }; - } - return { - kind: 'transient', - cause: synopsisResult.kind, - detail: synopsisResult.detail ?? 'transient', - }; + throw failure; + } + if (poolResult.aborted || args.abortSignal?.aborted) { + return { kind: 'transient', cause: 'timeout', detail: 'aborted' }; } // All chunks synthesized successfully. Single batch embed (D27 P2-2). @@ -528,6 +506,113 @@ async function tryBuildPhase1(opts: { } } +class ChunkSynopsisPhase1Error extends Error { + constructor(readonly result: Exclude) { + super(`chunk synopsis failed: ${result.kind}`); + this.name = 'ChunkSynopsisPhase1Error'; + } +} + +async function buildWrappedChunkText(opts: { + chunk: ChunkInput; + sourceText: string; + safeTitle: string; + page: Page; + args: ReembedPageArgs; + haikuModel: string; +}): Promise { + const { chunk: c, sourceText, safeTitle, page, args, haikuModel } = opts; + + // Code chunks always bypass the wrapper (D20-T4) — pass through. + if (c.chunk_source === 'fenced_code') { + return c.chunk_text; + } + + // Acquire rate-lease per chunk (D26 P0-3). Inline callers pass no + // hooks; only the Minion handler wires through rate-leases.ts. + let lease: unknown; + let leaseAcquired = false; + let synopsisResult: GeneratePerChunkSynopsisResult; + try { + if (args.acquireSynopsisLease) { + try { + lease = await args.acquireSynopsisLease(); + } catch (err) { + if (args.abortSignal?.aborted || isAbortError(err)) { + throw new ChunkSynopsisPhase1Error({ + kind: 'transient', + cause: 'timeout', + detail: 'aborted', + }); + } + throw err; + } + leaseAcquired = true; + } + synopsisResult = await generatePerChunkSynopsis({ + documentText: sourceText, + chunkText: c.chunk_text, + pageTitle: page.title, + pageSlug: args.pageSlug, + sourceId: args.sourceId, + chunkIndex: c.chunk_index, + model: haikuModel, + abortSignal: args.abortSignal, + }); + } finally { + if (leaseAcquired && args.releaseSynopsisLease) { + try { + await args.releaseSynopsisLease(lease); + } catch { + // Lease release failure shouldn't abort the page; surfacing it + // would race with the synopsis result. Audit-only. + } + } + } + + if (synopsisResult.kind === 'success') { + const prefix = buildContextualPrefix(safeTitle, synopsisResult.synopsis); + return wrapChunkForEmbedding(c.chunk_text, prefix, c.chunk_source); + } + + // Failure classification per D27 P1-2: + // refusal | empty | malformed → page-level fall-back to title-only + // auth_failure → permanent (won't fix with retry) + // rate_limit | timeout | network | provider_5xx → transient + // source_missing → walked into fallback already; would be 'malformed' + // from generatePerChunkSynopsis if we ever propagated it here + if ( + synopsisResult.kind === 'refusal' || + synopsisResult.kind === 'empty' || + synopsisResult.kind === 'malformed' + ) { + throw new ChunkSynopsisPhase1Error({ + kind: 'page_level_fallback_requested', + cause: synopsisResult.kind, + }); + } + if (synopsisResult.kind === 'auth_failure') { + throw new ChunkSynopsisPhase1Error({ + kind: 'permanent', + cause: synopsisResult.kind, + detail: synopsisResult.detail ?? 'auth failure', + }); + } + throw new ChunkSynopsisPhase1Error({ + kind: 'transient', + cause: synopsisResult.kind, + detail: synopsisResult.detail ?? 'transient', + }); +} + +function isAbortError(err: unknown): boolean { + return ( + typeof err === 'object' && + err !== null && + (err as { name?: unknown }).name === 'AbortError' + ); +} + /** * Source-text fallback chain per D11: * 1. read page.source_path from disk (truest "document") diff --git a/src/core/minions/handlers/contextual-reindex-per-chunk.ts b/src/core/minions/handlers/contextual-reindex-per-chunk.ts index 08b61ff4a..9d03b80f6 100644 --- a/src/core/minions/handlers/contextual-reindex-per-chunk.ts +++ b/src/core/minions/handlers/contextual-reindex-per-chunk.ts @@ -40,6 +40,7 @@ import { UnrecoverableError } from '../types.ts'; import type { BrainEngine } from '../../engine.ts'; import { reembedPageWithContextualRetrieval, + resolveContextualChunkConcurrency, type ReembedPageResult, } from '../../contextual-retrieval-service.ts'; import { @@ -132,7 +133,7 @@ export function makeContextualReindexHandler(opts: MakeContextualReindexHandlerO // call inside the service acquires/releases a lease against the // shared key across all worker processes. const maxConcurrent = resolveMaxConcurrent(); - let currentLeaseId: number | null = null; + const chunkConcurrency = resolveContextualChunkConcurrency(); const result: ReembedPageResult = await reembedPageWithContextualRetrieval({ engine, @@ -141,32 +142,32 @@ export function makeContextualReindexHandler(opts: MakeContextualReindexHandlerO globalMode, killSwitchDisabled, abortSignal: ctx.signal, + chunkConcurrency, acquireSynopsisLease: async () => { // Poll-acquire with brief backoff. The service's per-chunk loop - // is sequential within a page; this guards against the cross- - // worker pile-up. + // is bounded within a page; this guards against the cross-worker + // pile-up and remains the global rate governor. let attempts = 0; const maxAttempts = 60; // ~1 min max wait per chunk before giving up while (attempts < maxAttempts) { + if (ctx.signal.aborted) throw abortError(); const res = await acquireLease(engine, RATE_LEASE_KEY, ctx.id, maxConcurrent, { ttlMs: 60_000, }); if (res.acquired && res.leaseId != null) { - currentLeaseId = res.leaseId; - return; + return res.leaseId; } attempts++; - await new Promise((r) => setTimeout(r, 1000)); + await sleepWithAbort(1000, ctx.signal); } throw new Error( `Failed to acquire ${RATE_LEASE_KEY} lease after ${maxAttempts} attempts; ` + `Haiku rate limit pile-up too deep.`, ); }, - releaseSynopsisLease: async () => { - if (currentLeaseId != null) { - await releaseLease(engine, currentLeaseId); - currentLeaseId = null; + releaseSynopsisLease: async (lease) => { + if (typeof lease === 'number') { + await releaseLease(engine, lease); } }, }); @@ -218,6 +219,26 @@ async function tryLoadPageAcrossSources( return null; } +function sleepWithAbort(ms: number, signal: AbortSignal): Promise { + return new Promise((resolve, reject) => { + if (signal.aborted) { + reject(abortError()); + return; + } + const timer = setTimeout(resolve, ms); + signal.addEventListener('abort', () => { + clearTimeout(timer); + reject(abortError()); + }, { once: true }); + }); +} + +function abortError(): Error { + const err = new Error('aborted'); + err.name = 'AbortError'; + return err; +} + function classifyResult( pageSlug: string, result: ReembedPageResult, diff --git a/test/contextual-retrieval-service-pure.test.ts b/test/contextual-retrieval-service-pure.test.ts index 6918d41b3..3ba8d56d1 100644 --- a/test/contextual-retrieval-service-pure.test.ts +++ b/test/contextual-retrieval-service-pure.test.ts @@ -1,20 +1,38 @@ /** * Pure-function tests for src/core/contextual-retrieval-service.ts. * - * The full service test (PHASE 1 + PHASE 2 happy path, refusal restart, - * transient error propagation) needs a real PGLite + gateway stub seam. - * That lands in test/e2e/contextual-retrieval.test.ts. This file pins - * the service's pure helpers: corpus_generation hash composition + the - * expectedMode helper used by the T9 reindex sweep predicate. + * This file pins the service's pure helpers plus hermetic service behavior + * driven through fake engine + gateway seams. Full PGLite coverage lives in + * test/e2e/contextual-retrieval-pglite.test.ts. */ -import { describe, test, expect } from 'bun:test'; +import { afterEach, describe, test, expect } from 'bun:test'; import { computeCorpusGeneration, computeSourceTextHash, expectedModeForPageSourceOnly, + reembedPageWithContextualRetrieval, + resolveContextualChunkConcurrency, TITLE_WRAPPER_VERSION, } from '../src/core/contextual-retrieval-service.ts'; +import { + __setChatTransportForTests, + __setEmbedTransportForTests, + configureGateway, + resetGateway, + type ChatOpts, + type ChatResult, +} from '../src/core/ai/gateway.ts'; +import type { ChunkInput } from '../src/core/types.ts'; +import { withEnv } from './helpers/with-env.ts'; + +const TEST_DIMS = 1536; + +afterEach(() => { + __setChatTransportForTests(null); + __setEmbedTransportForTests(null); + resetGateway(); +}); describe('computeCorpusGeneration', () => { test('returns 16-char hex hash', () => { @@ -138,3 +156,317 @@ describe('expectedModeForPageSourceOnly (T9 reindex sweep helper)', () => { } }); }); + +describe('resolveContextualChunkConcurrency', () => { + test('defaults to 4 and reads the process env', async () => { + await withEnv({ GBRAIN_CONTEXTUAL_CHUNK_CONCURRENCY: undefined }, async () => { + expect(resolveContextualChunkConcurrency()).toBe(4); + }); + await withEnv({ GBRAIN_CONTEXTUAL_CHUNK_CONCURRENCY: '7' }, async () => { + expect(resolveContextualChunkConcurrency()).toBe(7); + }); + }); + + test('clamps to [1, 16] and ignores invalid values', () => { + expect(resolveContextualChunkConcurrency({ + GBRAIN_CONTEXTUAL_CHUNK_CONCURRENCY: '0', + })).toBe(1); + expect(resolveContextualChunkConcurrency({ + GBRAIN_CONTEXTUAL_CHUNK_CONCURRENCY: '-3', + })).toBe(1); + expect(resolveContextualChunkConcurrency({ + GBRAIN_CONTEXTUAL_CHUNK_CONCURRENCY: '99', + })).toBe(16); + expect(resolveContextualChunkConcurrency({ + GBRAIN_CONTEXTUAL_CHUNK_CONCURRENCY: '1.9', + })).toBe(1); + expect(resolveContextualChunkConcurrency({ + GBRAIN_CONTEXTUAL_CHUNK_CONCURRENCY: 'not-a-number', + })).toBe(4); + }); +}); + +describe('per-chunk synopsis concurrency', () => { + test('concurrency > 1 preserves chunk-order embed input', async () => { + const chunks = makeChunks(['alpha', 'beta', 'gamma', 'delta']); + const delays: Record = { alpha: 30, beta: 5, gamma: 20, delta: 1 }; + const sequential = await runWithChatStub({ + chunks, + concurrency: 1, + delayForChunk: (chunk) => delays[chunk] ?? 1, + }); + const parallel = await runWithChatStub({ + chunks, + concurrency: 4, + delayForChunk: (chunk) => delays[chunk] ?? 1, + }); + + expect(parallel.result.kind).toBe('success'); + expect(parallel.embedInputs).toEqual(sequential.embedInputs); + expect(parallel.embeddedChunks.map((c) => c.chunk_text)).toEqual( + chunks.map((c) => c.chunk_text), + ); + }); + + test('concurrency is bounded', async () => { + let active = 0; + let maxActive = 0; + let leaseActive = 0; + let maxLeaseActive = 0; + let acquired = 0; + let released = 0; + const chunks = makeChunks(Array.from({ length: 8 }, (_, i) => `chunk-${i}`)); + const out = await runWithChatStub({ + chunks, + concurrency: 3, + acquireSynopsisLease: async () => { + acquired++; + leaseActive++; + maxLeaseActive = Math.max(maxLeaseActive, leaseActive); + return acquired; + }, + releaseSynopsisLease: async () => { + released++; + leaseActive--; + }, + chat: async (opts) => { + active++; + maxActive = Math.max(maxActive, active); + try { + await delay(20, opts.abortSignal); + return chatSuccess(`Synopsis for ${extractChunk(opts)}`); + } finally { + active--; + } + }, + }); + + expect(out.result.kind).toBe('success'); + expect(maxActive).toBeGreaterThan(1); + expect(maxActive).toBeLessThanOrEqual(3); + expect(maxLeaseActive).toBeLessThanOrEqual(3); + expect(acquired).toBe(8); + expect(released).toBe(8); + expect(leaseActive).toBe(0); + }); + + test('one chunk failure aborts queued work and falls back at page level', async () => { + let started = 0; + const chunks = makeChunks(Array.from({ length: 9 }, (_, i) => `chunk-${i}`)); + const out = await runWithChatStub({ + chunks, + concurrency: 3, + chat: async (opts) => { + started++; + const chunk = extractChunk(opts); + if (chunk === 'chunk-0') return chatSuccess(''); + await delay(30, opts.abortSignal); + return chatSuccess(`Synopsis for ${chunk}`); + }, + }); + + expect(out.result.kind).toBe('page_fallback'); + expect(started).toBeLessThanOrEqual(3); + }); + + test('fenced code chunks bypass synopsis calls and leases', async () => { + let chatCalls = 0; + let leaseCalls = 0; + const chunks: ChunkInput[] = [ + { chunk_index: 0, chunk_text: 'intro', chunk_source: 'compiled_truth' }, + { chunk_index: 1, chunk_text: 'const x = 1;', chunk_source: 'fenced_code' }, + { chunk_index: 2, chunk_text: 'outro', chunk_source: 'compiled_truth' }, + ]; + + const out = await runWithChatStub({ + chunks, + concurrency: 3, + acquireSynopsisLease: async () => { + leaseCalls++; + }, + releaseSynopsisLease: async () => {}, + chat: async (opts) => { + chatCalls++; + return chatSuccess(`Synopsis for ${extractChunk(opts)}`); + }, + }); + + expect(out.result.kind).toBe('success'); + expect(chatCalls).toBe(2); + expect(leaseCalls).toBe(2); + expect(out.embedInputs[1]).toBe('const x = 1;'); + }); + + test('abortSignal cancels in-flight and queued synopsis work promptly', async () => { + const controller = new AbortController(); + let started = 0; + const chunks = makeChunks(Array.from({ length: 20 }, (_, i) => `chunk-${i}`)); + const startedAt = Date.now(); + const promise = runWithChatStub({ + chunks, + concurrency: 4, + abortSignal: controller.signal, + chat: async (opts) => { + started++; + await delay(1000, opts.abortSignal); + return chatSuccess(`Synopsis for ${extractChunk(opts)}`); + }, + }); + setTimeout(() => controller.abort(), 20); + + const out = await promise; + expect(out.result.kind).toBe('transient_error'); + if (out.result.kind === 'transient_error') { + expect(out.result.cause).toBe('timeout'); + } + expect(started).toBeLessThanOrEqual(4); + expect(Date.now() - startedAt).toBeLessThan(300); + }); +}); + +function makeChunks(texts: string[]): ChunkInput[] { + return texts.map((text, i) => ({ + chunk_index: i, + chunk_text: text, + chunk_source: 'compiled_truth', + })); +} + +async function runWithChatStub(opts: { + chunks: ChunkInput[]; + concurrency: number; + abortSignal?: AbortSignal; + delayForChunk?: (chunk: string) => number; + chat?: (opts: ChatOpts) => Promise; + acquireSynopsisLease?: () => Promise; + releaseSynopsisLease?: (lease?: unknown) => Promise; +}) { + configureGateway({ + embedding_model: 'openai:text-embedding-3-large', + embedding_dimensions: TEST_DIMS, + env: { OPENAI_API_KEY: 'sk-test' }, + }); + + const embedInputs: string[][] = []; + __setEmbedTransportForTests(async ({ values }: any) => { + embedInputs.push([...values]); + return { + embeddings: values.map((_: string, i: number) => + Array.from({ length: TEST_DIMS }, () => 0.001 + i * 0.001), + ), + usage: { tokens: 0 }, + } as any; + }); + + __setChatTransportForTests(opts.chat ?? (async (chatOpts) => { + const chunk = extractChunk(chatOpts); + await delay(opts.delayForChunk?.(chunk) ?? 1, chatOpts.abortSignal); + return chatSuccess(`Synopsis for ${chunk}`); + })); + + const engine = makeServiceEngine(opts.chunks); + const result = await reembedPageWithContextualRetrieval({ + engine, + pageSlug: 'wiki/concepts/concurrency-test', + sourceId: 'default', + globalMode: 'per_chunk_synopsis', + chunkConcurrency: opts.concurrency, + abortSignal: opts.abortSignal, + ...(opts.acquireSynopsisLease && { acquireSynopsisLease: opts.acquireSynopsisLease }), + ...(opts.releaseSynopsisLease && { releaseSynopsisLease: opts.releaseSynopsisLease }), + }); + + return { + result, + embedInputs: embedInputs.flat(), + embeddedChunks: engine.embeddedChunks as ChunkInput[], + }; +} + +function makeServiceEngine(chunks: ChunkInput[]) { + const engine: any = { + embeddedChunks: [] as ChunkInput[], + async getPage() { + return { + id: 1, + slug: 'wiki/concepts/concurrency-test', + source_id: 'default', + type: 'concept', + title: 'Concurrency Test', + compiled_truth: chunks.map((c) => c.chunk_text).join('\n\n'), + timeline: '', + frontmatter: {}, + created_at: new Date('2026-01-01T00:00:00Z'), + updated_at: new Date('2026-01-01T00:00:00Z'), + deleted_at: null, + }; + }, + async executeRaw() { + return [{ + id: 'default', + name: 'Default', + local_path: null, + last_commit: null, + last_sync_at: null, + config: {}, + created_at: new Date('2026-01-01T00:00:00Z'), + contextual_retrieval_mode: null, + trust_frontmatter_overrides: false, + }]; + }, + async getChunks() { + return chunks; + }, + async transaction(fn: (tx: any) => Promise) { + await fn({ + upsertChunks: async (_slug: string, embedded: ChunkInput[]) => { + engine.embeddedChunks = embedded; + }, + updatePageContextualRetrievalState: async () => {}, + }); + }, + async updatePageContextualRetrievalState() {}, + }; + return engine; +} + +function extractChunk(opts: ChatOpts): string { + const content = String(opts.messages[0]?.content ?? ''); + return content.match(/\n([\s\S]*?)\n<\/chunk>/)?.[1] ?? ''; +} + +function chatSuccess(text: string): ChatResult { + return { + text, + blocks: [], + stopReason: 'end', + usage: { + input_tokens: 0, + output_tokens: 0, + cache_read_tokens: 0, + cache_creation_tokens: 0, + }, + model: 'stub:chat', + providerId: 'stub', + }; +} + +function delay(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(abortError()); + return; + } + const timer = setTimeout(resolve, ms); + signal?.addEventListener('abort', () => { + clearTimeout(timer); + reject(abortError()); + }, { once: true }); + }); +} + +function abortError(): Error { + const err = new Error('aborted'); + err.name = 'AbortError'; + return err; +} From 97bdf6acc18e68dbb62688c45de8f87fc1b8fb48 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:35:09 -0700 Subject: [PATCH 33/39] fix(health): count 'entity' pages in graph health metrics (#2639) (#3330) Reland of #2639, reverted with its batch in 68e4cebd. getHealth's entity_pages CTE and the top-linked-pages query only match the legacy 'person' and 'company' types, so brains using the gbrain-base-v2 pack's 'entity' type report 0% entity link/timeline coverage in `gbrain health`. Add 'entity' to both queries in both engines (PGLite + Postgres, in lockstep per the engine-parity rule). Reland fix (the batch-red root cause): the original PR's test expected the entities/project-x page to appear in orphan_pages, but #3023's shared orphan-reporting policy (landed before #2639 merged) excludes the 'entities' first segment from orphan reporting, so the test failed on master. Orphan expectations now account for the policy exclusion. Co-authored-by: Tyler Robinson Co-authored-by: Claude Fable 5 --- src/core/pglite-engine.ts | 4 ++-- src/core/postgres-engine.ts | 4 ++-- test/pglite-engine.test.ts | 13 ++++++++----- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/core/pglite-engine.ts b/src/core/pglite-engine.ts index 0af4abc57..6d06bd320 100644 --- a/src/core/pglite-engine.ts +++ b/src/core/pglite-engine.ts @@ -5261,7 +5261,7 @@ export class PGLiteEngine implements BrainEngine { // dashboard, v0.10.3 metrics give entity-page-level granularity. const { rows: [h] } = await this.db.query(` WITH entity_pages AS ( - SELECT id, slug FROM pages WHERE type IN ('person', 'company') + SELECT id, slug FROM pages WHERE type IN ('entity', 'person', 'company') ) SELECT (SELECT count(*) FROM pages) as page_count, @@ -5289,7 +5289,7 @@ export class PGLiteEngine implements BrainEngine { SELECT p.slug, (SELECT count(*) FROM links l WHERE l.from_page_id = p.id OR l.to_page_id = p.id)::int as link_count FROM pages p - WHERE p.type IN ('person', 'company') + WHERE p.type IN ('entity', 'person', 'company') ORDER BY link_count DESC LIMIT 5 `); diff --git a/src/core/postgres-engine.ts b/src/core/postgres-engine.ts index 396c48737..7886afc70 100644 --- a/src/core/postgres-engine.ts +++ b/src/core/postgres-engine.ts @@ -5364,7 +5364,7 @@ export class PostgresEngine implements BrainEngine { // dashboard health. const [h] = await sql` WITH entity_pages AS ( - SELECT id, slug FROM pages WHERE type IN ('person', 'company') + SELECT id, slug FROM pages WHERE type IN ('entity', 'person', 'company') ) SELECT (SELECT count(*) FROM pages) as page_count, @@ -5389,7 +5389,7 @@ export class PostgresEngine implements BrainEngine { SELECT p.slug, (SELECT count(*) FROM links l WHERE l.from_page_id = p.id OR l.to_page_id = p.id)::int as link_count FROM pages p - WHERE p.type IN ('person', 'company') + WHERE p.type IN ('entity', 'person', 'company') ORDER BY link_count DESC LIMIT 5 `; diff --git a/test/pglite-engine.test.ts b/test/pglite-engine.test.ts index fe24f1463..e41b3c4f8 100644 --- a/test/pglite-engine.test.ts +++ b/test/pglite-engine.test.ts @@ -1434,6 +1434,7 @@ describe('PGLiteEngine: getHealth graph metrics', () => { await engine.putPage('people/alice', { ...testPage, type: 'person', title: 'Alice' }); await engine.putPage('people/bob', { ...testPage, type: 'person', title: 'Bob' }); await engine.putPage('companies/acme', { ...testPage, type: 'company', title: 'Acme' }); + await engine.putPage('entities/project-x', { ...testPage, type: 'entity', title: 'Project X' }); }); test('link_coverage = 0 when no links exist', async () => { @@ -1442,17 +1443,17 @@ describe('PGLiteEngine: getHealth graph metrics', () => { }); test('link_coverage = % of entity pages with >= 1 inbound link', async () => { - // Acme gets 1 inbound link (from Alice), Alice/Bob get 0 inbound. - // 1 of 3 entity pages has inbound links -> 33%. + // Acme gets 1 inbound link (from Alice); Alice/Bob/Project X get 0 inbound. + // 1 of 4 entity pages has inbound links -> 25%. await engine.addLink('people/alice', 'companies/acme', '', 'works_at'); const h = await engine.getHealth(); - expect(h.link_coverage).toBeCloseTo(1 / 3, 2); + expect(h.link_coverage).toBeCloseTo(1 / 4, 2); }); test('timeline_coverage = % with >= 1 timeline entry', async () => { await engine.addTimelineEntry('people/alice', { date: '2026-01-15', summary: 'Joined' }); const h = await engine.getHealth(); - expect(h.timeline_coverage).toBeCloseTo(1 / 3, 2); + expect(h.timeline_coverage).toBeCloseTo(1 / 4, 2); }); test('most_connected lists top entities by link count', async () => { @@ -1465,7 +1466,9 @@ describe('PGLiteEngine: getHealth graph metrics', () => { }); test('orphan_pages: pages with neither inbound nor outbound links', async () => { - // All 3 pages start with no links. Expect 3 orphans. + // All 4 pages start with no links, but entities/project-x is excluded + // from orphan reporting by the shared orphan policy ('entities' is a + // first-segment exclusion in orphan-policy.ts). Expect 3 orphans. const h = await engine.getHealth(); expect(h.orphan_pages).toBe(3); From cd252b080bf70aa450df427fa82b5cd5f6596712 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:35:13 -0700 Subject: [PATCH 34/39] fix(test): isolate GBRAIN_HOME in hybrid-reranker integration test (#1527) (#2640) (#3327) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The four `hybridSearch — reranker enabled (reorder)` cases stub the gateway at 1536 dims (DIMS). Since v0.36.3.0 hybridSearch resolves the embedding column via loadConfig(), whose precedence is cfg.embedding_dimensions > gateway dims > default. On any machine whose ~/.gbrain/config.json sets embedding_dimensions to something other than 1536 (e.g. text-embedding-3-small at 1280), the real config outranks the stub: the 1536-d stub vector fails the gateway dim check, the error is swallowed, search falls back to keyword-only, and the reranker never runs (rerankerFn gets 0 docs, rerank_score undefined). Green in CI only because a fresh runner has no config file — deterministic red on a contributor's machine. Fix (test-only): isolate GBRAIN_HOME to an empty tmpdir in beforeAll so loadConfig() returns null and the stub's dims win, then restore it and clean up in afterAll. Same idiom as emptyHome() in test/ai/gateway-probe-chat-model.test.ts. Verified with a planted ~/.gbrain/config.json at 1280 dims: 2 pass / 4 fail before, 6 pass / 0 fail after; still green with no config file. typecheck clean. Fixes #1527 Co-authored-by: Willisbest <132954469+Willisbest@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 --- ...hybrid-reranker-integration.serial.test.ts | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test/search/hybrid-reranker-integration.serial.test.ts b/test/search/hybrid-reranker-integration.serial.test.ts index 3ba02c371..34344731e 100644 --- a/test/search/hybrid-reranker-integration.serial.test.ts +++ b/test/search/hybrid-reranker-integration.serial.test.ts @@ -31,9 +31,24 @@ import { } from '../../src/core/ai/gateway.ts'; import type { PageInput, SearchOpts } from '../../src/core/types.ts'; import type { RerankInput, RerankResult } from '../../src/core/ai/gateway.ts'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; let engine: PGLiteEngine; +// These tests stub the gateway at 1536 dims (DIMS). Since v0.36.3.0 hybridSearch +// resolves the embedding column via loadConfig(), whose precedence is +// cfg.embedding_dimensions > gateway dims > default — so a contributor's real +// ~/.gbrain/config.json (e.g. text-embedding-3-small at 1280) outranks the stub, +// the 1536-d stub vector then fails the gateway dim check, search silently falls +// back to keyword-only, and the reranker never runs (0 docs → 4 tests fail). CI +// is green only because a fresh runner has no config file (#1527). Isolate +// GBRAIN_HOME to an empty tmpdir so loadConfig() returns null and the stub's dims +// win — same idiom as emptyHome() in test/ai/gateway-probe-chat-model.test.ts. +let prevGbrainHome: string | undefined; +let isolatedHome: string; + const DIMS = 1536; // gateway default embedding dim const FAKE_EMB = Array.from({ length: DIMS }, (_, j) => (j === 0 ? 1 : 0.01)); @@ -44,6 +59,12 @@ function stubEmbeddings(): void { } beforeAll(async () => { + // Hermetic config home: ignore the machine's real ~/.gbrain so its + // embedding_dimensions can't outrank the 1536-d stub (see note above, #1527). + prevGbrainHome = process.env.GBRAIN_HOME; + isolatedHome = mkdtempSync(join(tmpdir(), 'gbrain-rerank-home-')); + process.env.GBRAIN_HOME = isolatedHome; + engine = new PGLiteEngine(); await engine.connect({}); await engine.initSchema(); @@ -106,6 +127,9 @@ afterAll(async () => { __setEmbedTransportForTests(null); resetGateway(); await engine.disconnect(); + if (prevGbrainHome === undefined) delete process.env.GBRAIN_HOME; + else process.env.GBRAIN_HOME = prevGbrainHome; + rmSync(isolatedHome, { recursive: true, force: true }); }); describe('hybridSearch — reranker disabled (pass-through)', () => { From f5e5736f09057a519e145eac90e88531cf5fd6e1 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:35:18 -0700 Subject: [PATCH 35/39] feat(ai): dashscope-rerank recipe (DashScope serves PLURAL /reranks under compatible-api) (#2644) (#3328) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DashScope's OpenAI-compatible rerank endpoint lives at {base}/compatible-api/v1/reranks — PLURAL leaf, different base path from the embedding surface (compatible-mode). Reusing llama-server-reranker against DashScope forces users to hand-patch the recipe's '/rerank' leaf in node_modules, which every upgrade silently reverts (and llama.cpp genuinely serves singular /rerank, so changing that recipe would break real llama.cpp users). New dedicated recipe rides the v0.40.6.1 recipe-pluggable reranker path: - id dashscope-rerank, base_url_default compatible-api/v1 (intl), ZE wire - path '/reranks', default_timeout_ms 30s, 5MB payload ceiling - models: only qwen3-rerank (live-verified 200; gte-rerank-v2 is rejected by the compat surface with 'Unsupported model for OpenAI compatibility mode', so it is deliberately not listed) - separate recipe (not a reranker touchpoint on dashscope) because provider_base_urls is keyed by recipe id and the two capabilities need different prefixes — same topology as llama-server vs llama-server-reranker Tests: recipe shape smoke mirroring recipe-llama-server-reranker.test.ts (path/timeout/payload pins, /v1/v1 concat guard, auth resolve, sibling recipe isolation). bun test test/ai/: 322 pass / 0 fail. Co-authored-by: Yicon Co-authored-by: Claude Opus 4.8 --- src/core/ai/recipes/dashscope-rerank.ts | 61 +++++++++++++++++++ src/core/ai/recipes/index.ts | 2 + test/ai/recipe-dashscope-rerank.test.ts | 79 +++++++++++++++++++++++++ 3 files changed, 142 insertions(+) create mode 100644 src/core/ai/recipes/dashscope-rerank.ts create mode 100644 test/ai/recipe-dashscope-rerank.test.ts diff --git a/src/core/ai/recipes/dashscope-rerank.ts b/src/core/ai/recipes/dashscope-rerank.ts new file mode 100644 index 000000000..155c20b56 --- /dev/null +++ b/src/core/ai/recipes/dashscope-rerank.ts @@ -0,0 +1,61 @@ +import type { Recipe } from '../types.ts'; + +/** + * Alibaba DashScope (灵积) reranker. DashScope's OpenAI-compatible surface + * splits by capability: embeddings live under `/compatible-mode/v1` (see the + * sibling `dashscope` recipe) while rerank lives under `/compatible-api/v1` + * with a PLURAL leaf — `POST {base}/reranks`. Wire shape matches ZeroEntropy: + * request `{model, query, documents, top_n?}`, response + * `{results: [{index, relevance_score}]}` — so it rides gateway.rerank()'s + * native path with only the recipe-pluggable `path` override (v0.40.6.1). + * + * This is a SEPARATE recipe rather than a reranker touchpoint on `dashscope` + * because the two capabilities need different base URLs (`compatible-mode` + * vs `compatible-api`) and `provider_base_urls` is keyed by recipe id — one + * recipe can't point embeddings and rerank at different prefixes. Same + * topology precedent as llama-server vs llama-server-reranker. + * + * Live-verified against the China endpoint (2026-07): `/reranks` with + * `qwen3-rerank` → 200 `results[].relevance_score`; `/rerank` (singular) + * → 404; `gte-rerank-v2` → 404 "Unsupported model for OpenAI compatibility + * mode" (native-API only, so it is deliberately NOT listed here). + * + * Note: the international endpoint requires a region-aware DASHSCOPE_API_KEY. + * China-region users point at https://dashscope.aliyuncs.com/compatible-api/v1 + * via `provider_base_urls['dashscope-rerank']`, mirroring the embedding + * recipe's convention. + */ +export const dashscopeRerank: Recipe = { + id: 'dashscope-rerank', + name: 'Alibaba DashScope (灵积, reranker)', + tier: 'openai-compat', + implementation: 'openai-compatible', + base_url_default: 'https://dashscope-intl.aliyuncs.com/compatible-api/v1', + auth_env: { + required: ['DASHSCOPE_API_KEY'], + setup_url: 'https://help.aliyun.com/zh/model-studio/getting-started/', + }, + touchpoints: { + reranker: { + // Only the model verified live on the OpenAI-compat /reranks surface. + // gte-rerank-v2 exists on DashScope's native API but the compat path + // rejects it ("Unsupported model for OpenAI compatibility mode"). + models: ['qwen3-rerank'], + default_model: 'qwen3-rerank', + // Mirror ZE's defensive per-request ceiling; gateway.rerank() + // pre-flights body size and fails open. + max_payload_bytes: 5_000_000, + // PLURAL leaf under compatible-api — the whole reason this recipe + // exists. `${base_url}${path}` → `…/compatible-api/v1/reranks`. + path: '/reranks', + // Hosted API: no local warmup, but cross-region latency can exceed + // the 5s gateway default (same rationale as llama-server-reranker). + default_timeout_ms: 30_000, + }, + }, + setup_hint: + 'Get an API key at https://help.aliyun.com/zh/model-studio/getting-started/, then ' + + '`export DASHSCOPE_API_KEY=...` and `gbrain config set search.reranker.model ' + + 'dashscope-rerank:qwen3-rerank`. China-region accounts: `gbrain config set ' + + 'provider_base_urls.dashscope-rerank https://dashscope.aliyuncs.com/compatible-api/v1`.', +}; diff --git a/src/core/ai/recipes/index.ts b/src/core/ai/recipes/index.ts index a91010291..49175f2ca 100644 --- a/src/core/ai/recipes/index.ts +++ b/src/core/ai/recipes/index.ts @@ -20,6 +20,7 @@ import { together } from './together.ts'; import { llamaServer } from './llama-server.ts'; import { minimax } from './minimax.ts'; import { dashscope } from './dashscope.ts'; +import { dashscopeRerank } from './dashscope-rerank.ts'; import { zhipu } from './zhipu.ts'; import { azureOpenAI } from './azure-openai.ts'; import { zeroentropyai } from './zeroentropyai.ts'; @@ -45,6 +46,7 @@ const ALL: Recipe[] = [ llamaServerReranker, minimax, dashscope, + dashscopeRerank, zhipu, azureOpenAI, zeroentropyai, diff --git a/test/ai/recipe-dashscope-rerank.test.ts b/test/ai/recipe-dashscope-rerank.test.ts new file mode 100644 index 000000000..ef009064b --- /dev/null +++ b/test/ai/recipe-dashscope-rerank.test.ts @@ -0,0 +1,79 @@ +/** + * dashscope-rerank recipe smoke. + * + * Sibling of recipe-llama-server-reranker.test.ts. Pins the recipe shape so: + * - id + tier + implementation + base_url stay byte-stable + * - reranker touchpoint declares the PLURAL `/reranks` leaf (the whole + * reason this recipe exists — DashScope's compatible-api surface 404s + * on singular `/rerank`) + `default_timeout_ms` + * - only live-verified models are listed (gte-rerank-v2 is native-API only + * and rejected by the OpenAI-compat surface) + */ + +import { describe, expect, test } from 'bun:test'; +import { getRecipe } from '../../src/core/ai/recipes/index.ts'; +import { defaultResolveAuth } from '../../src/core/ai/gateway.ts'; +import { AIConfigError } from '../../src/core/ai/errors.ts'; + +describe('recipe: dashscope-rerank', () => { + test('registered with expected shape', () => { + const r = getRecipe('dashscope-rerank'); + expect(r).toBeDefined(); + expect(r!.id).toBe('dashscope-rerank'); + expect(r!.tier).toBe('openai-compat'); + expect(r!.implementation).toBe('openai-compatible'); + expect(r!.base_url_default).toBe( + 'https://dashscope-intl.aliyuncs.com/compatible-api/v1', + ); + expect(r!.auth_env?.required).toEqual(['DASHSCOPE_API_KEY']); + }); + + test('declares reranker touchpoint with PLURAL /reranks path + timeout', () => { + const r = getRecipe('dashscope-rerank')!; + const tp = r.touchpoints.reranker; + expect(tp).toBeDefined(); + expect(tp!.path).toBe('/reranks'); + expect(tp!.default_timeout_ms).toBe(30_000); + expect(tp!.max_payload_bytes).toBe(5_000_000); + }); + + test('base_url + path concatenation produces /v1/reranks, NOT /v1/v1/…', () => { + const r = getRecipe('dashscope-rerank')!; + const combined = + r.base_url_default!.replace(/\/$/, '') + (r.touchpoints.reranker!.path ?? '/models/rerank'); + expect(combined).toBe('https://dashscope-intl.aliyuncs.com/compatible-api/v1/reranks'); + expect(combined).not.toContain('/v1/v1/'); + expect(combined.endsWith('/reranks')).toBe(true); + }); + + test('lists only the live-verified compat-surface model', () => { + const r = getRecipe('dashscope-rerank')!; + const tp = r.touchpoints.reranker!; + expect(tp.models).toEqual(['qwen3-rerank']); + expect(tp.default_model).toBe('qwen3-rerank'); + // gte-rerank-v2 is native-API only; the compat surface rejects it. + expect(tp.models).not.toContain('gte-rerank-v2'); + }); + + test('default auth: DASHSCOPE_API_KEY set → Bearer token', () => { + const r = getRecipe('dashscope-rerank')!; + const auth = defaultResolveAuth( + r, + { DASHSCOPE_API_KEY: 'sk-dashscope-fake' }, + 'reranker', + ); + expect(auth.headerName).toBe('Authorization'); + expect(auth.token).toBe('Bearer sk-dashscope-fake'); + }); + + test('default auth: missing DASHSCOPE_API_KEY → AIConfigError', () => { + const r = getRecipe('dashscope-rerank')!; + expect(() => defaultResolveAuth(r, {}, 'reranker')).toThrow(AIConfigError); + }); + + test('does not perturb the sibling dashscope embedding recipe', () => { + const emb = getRecipe('dashscope')!; + expect(emb.base_url_default).toBe('https://dashscope-intl.aliyuncs.com/compatible-mode/v1'); + expect(emb.touchpoints.reranker).toBeUndefined(); + }); +}); From 5e665c1c06649ede6ecc61ce125271fd42420c92 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:35:22 -0700 Subject: [PATCH 36/39] fix: clarify PGLite data-dir lock contention (#2658) (#3336) Co-authored-by: zay --- src/core/pglite-lock.ts | 47 ++++++++++++++++++++++++++-------------- test/pglite-lock.test.ts | 20 +++++++++++++++++ 2 files changed, 51 insertions(+), 16 deletions(-) diff --git a/src/core/pglite-lock.ts b/src/core/pglite-lock.ts index 74c91aa45..188e5c666 100644 --- a/src/core/pglite-lock.ts +++ b/src/core/pglite-lock.ts @@ -124,6 +124,35 @@ function isProcessAlive(pid: number): boolean { } } +function formatLockTimestamp(value: unknown): string { + return typeof value === 'number' && Number.isFinite(value) + ? new Date(value).toISOString() + : 'unknown time'; +} + +function pgliteLockTimeoutError(lockDir: string): Error { + const lockPath = join(lockDir, LOCK_FILE); + try { + const lockData = JSON.parse(readFileSync(lockPath, 'utf-8')); + const pid = String(lockData.pid ?? 'unknown'); + const command = String(lockData.command ?? 'unknown'); + const serveHint = command.includes('gbrain serve') + ? ' The holder looks like `gbrain serve`, so this is probably serve↔sync contention from an MCP/HTTP server; stop that server/client and rerun the command.' + : ''; + + return new Error( + `GBrain: Timed out waiting for PGLite data-dir lock. Process ${pid} has held it since ${formatLockTimestamp(lockData.acquired_at)} (command: ${command}). ` + + `Lock directory: ${lockDir}. If that process is dead, remove the lock directory and try again. ` + + `This is a PGLite data-dir lock, not the \`gbrain-sync:*\` advisory lock; \`gbrain sync --break-lock\` will not clear a live PGLite holder.` + + serveHint, + ); + } catch { + return new Error( + `GBrain: Timed out waiting for PGLite lock. Remove ${lockDir} and try again.` + ); + } +} + /** * Attempt to acquire an exclusive lock on the PGLite data directory. * Returns { acquired: true } if the lock was obtained, { acquired: false } otherwise. @@ -206,28 +235,14 @@ export async function acquireLock(dataDir: string | undefined, opts?: { timeoutM // mkdir failed — someone else grabbed it between our check and mkdir // This is fine, we'll retry if (Date.now() - startTime >= timeoutMs) { - // Timeout — report which process holds the lock - const lockPath = join(lockDir, LOCK_FILE); - try { - const lockData = JSON.parse(readFileSync(lockPath, 'utf-8')); - throw new Error( - `GBrain: Timed out waiting for PGLite lock. Process ${lockData.pid} has held it since ${new Date(lockData.acquired_at).toISOString()} (command: ${lockData.command}). ` + - `If that process is dead, remove ${lockDir} and try again.` - ); - } catch (readErr) { - if (readErr instanceof Error && readErr.message.startsWith('GBrain')) throw readErr; - throw new Error( - `GBrain: Timed out waiting for PGLite lock. Remove ${lockDir} and try again.` - ); - } + throw pgliteLockTimeoutError(lockDir); } // Brief wait before retry await new Promise(r => setTimeout(r, 500)); } } - // Should not reach here, but just in case - throw new Error(`GBrain: Timed out waiting for PGLite lock.`); + throw pgliteLockTimeoutError(lockDir); } /** diff --git a/test/pglite-lock.test.ts b/test/pglite-lock.test.ts index 2850a0a94..0f8c545a1 100644 --- a/test/pglite-lock.test.ts +++ b/test/pglite-lock.test.ts @@ -212,6 +212,26 @@ describe('pglite-lock #2058 heartbeat + steal-grace', () => { expect(existsSync(join(TEST_DIR, '.gbrain-lock'))).toBe(true); }); + test('explains live gbrain serve contention is not a sync advisory lock', async () => { + writeHolder({ + pid: process.pid, + acquiredAgoMs: 60_000, + refreshedAgoMs: 0, + command: 'bun /Users/master/.bun/bin/gbrain serve', + }); + + let message = ''; + try { + await acquireLock(TEST_DIR, { timeoutMs: 100 }); + } catch (error) { + message = error instanceof Error ? error.message : String(error); + } + expect(message).toContain('serve↔sync contention'); + expect(message).toContain('not the `gbrain-sync:*` advisory lock'); + expect(message).toContain('`gbrain sync --break-lock` will not clear a live PGLite holder'); + expect(existsSync(join(TEST_DIR, '.gbrain-lock'))).toBe(true); + }); + test('[REGRESSION] releaseLock does NOT remove a lock that was stolen + re-acquired by another process', async () => { // We acquire, then simulate a steal: another process reaped us past grace // and now owns the lock (different pid + acquired_at). Our releaseLock must From e1919fab9f718bd391b9e3f74e436c71edb2c23b Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:48:22 -0700 Subject: [PATCH 37/39] reland: fix(embed): stamp gateway-resolved model in content_chunks.model, not compiled default (#2846) (#3343) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(embed): stamp gateway-resolved model in content_chunks.model, not compiled default (#2846) upsertChunks fell back to the compile-time DEFAULT_EMBEDDING_MODEL ('zeroentropyai:zembed-1') when a ChunkInput carried no explicit `model`. The embed pipeline (src/commands/embed.ts) builds ChunkInputs without a `model` field, so rows whose vectors were produced by the config-resolved model (e.g. openai:text-embedding-3-large) were mislabeled with the hardcoded default — corrupting the provenance that signature-drift staleness and dimension-migration logic depend on. Both engines now resolve the gateway's runtime embedding model once per upsert and use it as the fallback, mirroring the existing resolve-then- default pattern used for schema sizing. Regression test added (pglite); verified via negative control that it fails against the old fallback. This is a write-path change (upsertChunks), not a search-path change, so retrieval eval replay is not applicable. Co-authored-by: Claude Opus 4.8 * test: Lane A.7 pins gateway-resolved chunk model, not compiled default #2846 changed upsertChunks' fallback from DEFAULT_EMBEDDING_MODEL to the gateway-resolved runtime model. Lane A.7 still pinned the old fallback, and the test preload (test/helpers/legacy-embedding-preload.ts) pins the gateway to openai:text-embedding-3-large for every test process — so the original #2846 landing failed this test deterministically and got batch- reverted. The test now asserts the resolved model (the intended #2846 semantics) while keeping the CDX2-4 bare-literal regression guard. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: SailorJoe6 Co-authored-by: Claude Opus 4.8 Co-authored-by: Garry Tan --- src/core/pglite-engine.ts | 14 ++++++++- src/core/postgres-engine.ts | 19 +++++++++++- test/e2e/embedding-column-pglite.test.ts | 38 ++++++++++++++++++++++++ test/v0_37_gap_fill.serial.test.ts | 23 +++++++++----- 4 files changed, 84 insertions(+), 10 deletions(-) diff --git a/src/core/pglite-engine.ts b/src/core/pglite-engine.ts index 6d06bd320..bae0181b2 100644 --- a/src/core/pglite-engine.ts +++ b/src/core/pglite-engine.ts @@ -2312,6 +2312,18 @@ export class PGLiteEngine implements BrainEngine { const params: unknown[] = []; let paramIdx = 1; + // Provenance fallback for chunks without an explicit `model`: resolve the + // gateway's runtime model, not the compile-time DEFAULT_EMBEDDING_MODEL. + // See postgres-engine.ts _upsertChunksOnce for the full rationale — pglite + // mirrors it for parity. + let resolvedModel: string = DEFAULT_EMBEDDING_MODEL; + try { + const gw = await import('./ai/gateway.ts'); + resolvedModel = gw.getEmbeddingModel() || resolvedModel; + } catch { + // Gateway unconfigured (unit tests / pre-connect): keep the default. + } + for (const chunk of chunks) { const embeddingStr = chunk.embedding ? '[' + Array.from(chunk.embedding).join(',') + ']' @@ -2344,7 +2356,7 @@ export class PGLiteEngine implements BrainEngine { if (embeddingImageStr) params.push(embeddingImageStr); params.push( pageId, chunk.chunk_index, chunk.chunk_text, chunk.chunk_source, - chunk.model || DEFAULT_EMBEDDING_MODEL, chunk.token_count || null, + chunk.model || resolvedModel, chunk.token_count || null, chunk.language || null, chunk.symbol_name || null, chunk.symbol_type || null, chunk.start_line ?? null, chunk.end_line ?? null, parentPath, chunk.doc_comment || null, chunk.symbol_name_qualified || null, diff --git a/src/core/postgres-engine.ts b/src/core/postgres-engine.ts index 7886afc70..ad119a5b9 100644 --- a/src/core/postgres-engine.ts +++ b/src/core/postgres-engine.ts @@ -2438,6 +2438,23 @@ export class PostgresEngine implements BrainEngine { const params: unknown[] = []; let paramIdx = 1; + // Provenance fallback for chunks that don't carry an explicit `model`: + // resolve the model the gateway ACTUALLY uses at runtime, not the + // compile-time DEFAULT_EMBEDDING_MODEL constant. Callers like `embed` + // build ChunkInputs without a `model` field (src/commands/embed.ts), so + // the old `chunk.model || DEFAULT_EMBEDDING_MODEL` fallback stamped the + // hardcoded default (e.g. zeroentropyai:zembed-1) onto rows whose vectors + // were produced by a different, config-resolved model — corrupting the + // provenance that signature-drift staleness + dim-migration logic trust. + // Mirrors the resolve-then-fallback pattern used for schema sizing above. + let resolvedModel: string = DEFAULT_EMBEDDING_MODEL; + try { + const gw = await import('./ai/gateway.ts'); + resolvedModel = gw.getEmbeddingModel() || resolvedModel; + } catch { + // Gateway unconfigured (unit tests / pre-connect): keep the default. + } + for (const chunk of chunks) { const embeddingStr = chunk.embedding ? '[' + Array.from(chunk.embedding).join(',') + ']' @@ -2467,7 +2484,7 @@ export class PostgresEngine implements BrainEngine { if (embeddingImageStr) params.push(embeddingImageStr); params.push( pageId, chunk.chunk_index, chunk.chunk_text, chunk.chunk_source, - chunk.model || DEFAULT_EMBEDDING_MODEL, chunk.token_count || null, + chunk.model || resolvedModel, chunk.token_count || null, chunk.language || null, chunk.symbol_name || null, chunk.symbol_type || null, chunk.start_line ?? null, chunk.end_line ?? null, parentPath, chunk.doc_comment || null, chunk.symbol_name_qualified || null, diff --git a/test/e2e/embedding-column-pglite.test.ts b/test/e2e/embedding-column-pglite.test.ts index 86ba41693..7254806cf 100644 --- a/test/e2e/embedding-column-pglite.test.ts +++ b/test/e2e/embedding-column-pglite.test.ts @@ -216,6 +216,44 @@ describe('hybridSearch + resolver — unknown column at entry (D11)', () => { }); }); +describe('upsertChunks — model provenance uses gateway-resolved model, not compiled default', () => { + // Regression (zbrain-rfi): when a caller builds ChunkInputs without an + // explicit `model` (as src/commands/embed.ts does), the engine used to + // stamp the compile-time DEFAULT_EMBEDDING_MODEL ('zeroentropyai:zembed-1') + // onto content_chunks.model — even though the vector was produced by the + // config-resolved model. That corrupted provenance the signature-drift + + // dim-migration logic trusts. The engine must fall back to the model the + // gateway ACTUALLY resolves at write time. + test('unspecified chunk.model records the resolved model, not zeroentropyai:zembed-1', async () => { + configureGateway({ + embedding_model: 'openai:text-embedding-3-large', + embedding_dimensions: 1536, + env: { OPENAI_API_KEY: 'sk-test' }, + }); + + await engine.putPage('docs/provenance-page', { + type: 'concept', + title: 'Provenance test page', + compiled_truth: 'Chunk whose model column must reflect the resolved model.', + }); + // No `model` field on the input — the write-side fallback must fill it. + await engine.upsertChunks('docs/provenance-page', [ + { chunk_index: 0, chunk_text: 'provenance chunk', chunk_source: 'compiled_truth' }, + ]); + + const rows = await engine.executeRaw<{ model: string }>( + `SELECT cc.model FROM content_chunks cc + JOIN pages p ON p.id = cc.page_id + WHERE p.slug = 'docs/provenance-page'`, + ); + expect(rows.length).toBe(1); + expect(rows[0].model).toBe('openai:text-embedding-3-large'); + expect(rows[0].model).not.toBe('zeroentropyai:zembed-1'); + + resetGateway(); + }); +}); + describe('buildVectorCastFragment — engine SQL composer (D3)', () => { test('vector descriptor emits $1::vector', () => { const r: ResolvedColumn = { diff --git a/test/v0_37_gap_fill.serial.test.ts b/test/v0_37_gap_fill.serial.test.ts index c1f275394..022256d21 100644 --- a/test/v0_37_gap_fill.serial.test.ts +++ b/test/v0_37_gap_fill.serial.test.ts @@ -30,11 +30,15 @@ import { configureGateway, resetGateway, __setEmbedTransportForTests } from '../ import { withEnv } from './helpers/with-env.ts'; // ───────────────────────────────────────────────────────────────────── -// Lane A.7 — Chunk-row INSERT model default tracks defaults.ts constant -// (not stale OpenAI literal). Pre-fix `chunk.model || 'text-embedding-3-large'` -// in both engines; post-fix `chunk.model || DEFAULT_EMBEDDING_MODEL`. +// Lane A.7 — Chunk-row INSERT model default tracks the gateway-resolved +// model (not a stale OpenAI literal, not the compile-time constant). +// Pre-fix `chunk.model || 'text-embedding-3-large'` in both engines; +// v0.37 fix `chunk.model || DEFAULT_EMBEDDING_MODEL`; #2846 tightened it +// to the gateway's runtime model so provenance matches the vector's +// actual producer (falls back to DEFAULT_EMBEDDING_MODEL only when the +// gateway is unconfigured). // ───────────────────────────────────────────────────────────────────── -describe('Lane A.7 — chunk-row INSERT default tracks ai/defaults.ts constant', () => { +describe('Lane A.7 — chunk-row INSERT default tracks the gateway-resolved model', () => { let engine: PGLiteEngine; beforeAll(async () => { @@ -47,8 +51,11 @@ describe('Lane A.7 — chunk-row INSERT default tracks ai/defaults.ts constant', await engine.disconnect(); }); - test('upsertChunks without explicit model: row stores DEFAULT_EMBEDDING_MODEL', async () => { - const { DEFAULT_EMBEDDING_MODEL } = await import('../src/core/ai/defaults.ts'); + test('upsertChunks without explicit model: row stores the gateway-resolved model', async () => { + // The test preload (test/helpers/legacy-embedding-preload.ts) pins the + // gateway to 'openai:text-embedding-3-large', so that's what the write + // site must stamp — NOT the compile-time DEFAULT_EMBEDDING_MODEL (#2846). + const { getEmbeddingModel } = await import('../src/core/ai/gateway.ts'); await engine.putPage('test/a7', { type: 'note', title: 'A.7', compiled_truth: 'hello' }); await engine.upsertChunks('test/a7', [ { chunk_index: 0, chunk_text: 'hello', chunk_source: 'compiled_truth' }, @@ -57,9 +64,9 @@ describe('Lane A.7 — chunk-row INSERT default tracks ai/defaults.ts constant', const rows = await engine.executeRaw<{ model: string }>( `SELECT model FROM content_chunks WHERE chunk_index = 0 LIMIT 1`, ); - expect(rows[0]?.model).toBe(DEFAULT_EMBEDDING_MODEL); + expect(rows[0]?.model).toBe(getEmbeddingModel()); // CDX2-4 regression: would have been 'text-embedding-3-large' - // (a literal pre-fix; production write site that was never tested). + // (a bare literal pre-fix; provider-prefixed form is required). expect(rows[0]?.model).not.toBe('text-embedding-3-large'); }); }); From d15e2ab8cf592601dd2ed0d0c93c302e2dae5603 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:48:27 -0700 Subject: [PATCH 38/39] fix(webhook): extract links for incremental push syncs (#2850) (#3337) * test(webhook): pin sync extraction contract (#2849) * test(webhook): target the submitted sync payload (#2849) * fix(webhook): run extraction in sync job (#2849) * fix(sync): align push trigger extraction (#2849) Co-authored-by: Song --- src/commands/serve-http.ts | 7 +++++-- src/commands/sync.ts | 1 + test/sources-webhook.test.ts | 23 +++++++++++++++++++++++ test/sync-trigger-cli.test.ts | 1 + 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/commands/serve-http.ts b/src/commands/serve-http.ts index 60c4f2ee2..5e30794f5 100644 --- a/src/commands/serve-http.ts +++ b/src/commands/serve-http.ts @@ -2249,8 +2249,10 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption // Other event types (ping, pull_request, etc.) return 202 'ignored' // so GitHub doesn't retry. // D15.5: HMAC compare uses the shared safeHexEqual helper. - // D18: submits 'sync' job with auto_embed_backfill=true and priority -10 - // (above autopilot's 0). + // D18: submits 'sync' job with extraction + auto_embed_backfill enabled and + // priority -10 (above autopilot's 0). This opts normal incremental pushes + // into sync's inline extraction while pagesAffected still identifies the + // changed pages. The sync core can still defer large (>100) changes. // --------------------------------------------------------------------------- const githubWebhookLimiter = rateLimit({ windowMs: 60_000, @@ -2370,6 +2372,7 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption 'sync', { sourceId: source.id, + noExtract: false, auto_embed_backfill: true, embed_reason: 'webhook', }, diff --git a/src/commands/sync.ts b/src/commands/sync.ts index 28c359441..d7e540c35 100644 --- a/src/commands/sync.ts +++ b/src/commands/sync.ts @@ -1423,6 +1423,7 @@ See also: { sourceId: sourceIdArg, repoPath: source.local_path, + noExtract: false, auto_embed_backfill: true, embed_reason: 'sync_trigger', }, diff --git a/test/sources-webhook.test.ts b/test/sources-webhook.test.ts index fdda0ece9..e8fd75b40 100644 --- a/test/sources-webhook.test.ts +++ b/test/sources-webhook.test.ts @@ -16,6 +16,7 @@ */ import { describe, test, expect } from 'bun:test'; import { createHmac } from 'node:crypto'; +import { readFileSync } from 'node:fs'; import { safeHexEqual } from '../src/core/timing-safe.ts'; const GITHUB_SECRET = 'super-secret-webhook-key'; @@ -123,3 +124,25 @@ describe('Branch ref construction (D5)', () => { expect(pushedRef === `refs/heads/${trackedBranch}`).toBe(false); }); }); + +describe('Webhook sync job extraction contract', () => { + test('opts into extraction before the pushed commit is consumed', () => { + const serveSource = readFileSync( + new URL('../src/commands/serve-http.ts', import.meta.url), + 'utf8', + ); + const routeStart = serveSource.indexOf("'/webhooks/github'"); + const queueStart = serveSource.indexOf('const job = await queue.add(', routeStart); + const responseStart = serveSource.indexOf('res.status(202)', queueStart); + expect(routeStart).toBeGreaterThanOrEqual(0); + expect(queueStart).toBeGreaterThan(routeStart); + expect(responseStart).toBeGreaterThan(queueStart); + + const routeSource = serveSource.slice(queueStart, responseStart); + const payload = routeSource.match( + /queue\.add\(\s*'sync',\s*\{([\s\S]*?)\}\s*,\s*\{/, + ); + expect(payload).not.toBeNull(); + expect(payload?.[1]).toMatch(/\bnoExtract:\s*false\b/); + }); +}); diff --git a/test/sync-trigger-cli.test.ts b/test/sync-trigger-cli.test.ts index ff31f7b45..07801b4ea 100644 --- a/test/sync-trigger-cli.test.ts +++ b/test/sync-trigger-cli.test.ts @@ -100,6 +100,7 @@ describe('runSyncTrigger', () => { const job = jobs[0]; expect(job.priority).toBe(-10); expect((job.data as { sourceId: string }).sourceId).toBe('default'); + expect((job.data as { noExtract: boolean }).noExtract).toBe(false); expect((job.data as { auto_embed_backfill: boolean }).auto_embed_backfill).toBe(true); }); From 1f319e6d5aff7674d8f48f289768ff75911a9ea8 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:48:34 -0700 Subject: [PATCH 39/39] fix(scripts): capture check/shard rc before watchdog teardown in no-timeout fallback (#2864) (#3340) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On machines with neither gtimeout nor timeout on PATH, run-verify-parallel.sh and run-unit-parallel.sh fall back to a bg-pid + sleep-watchdog cap. Both read $? only after tearing the watchdog down (kill + wait on cap_pid), so the sentinel .exit files recorded the killed watchdog's status — 143 — instead of the check/shard's own exit code. Every run reported total failure (verify: pass=0 fail=31; unit: rc=143 per shard) while every per-check/shard log showed success. Capture rc immediately after `wait $pid` in both scripts, and reap the watchdog's sleep child (pkill -P, children-first — the same orphan quirk the heartbeat cleanup documents) so the fallback stops leaking one sleep per check/shard. Regression tests force the fallback branch hermetically on any host via a curated PATH with no timeout binaries: the verify dispatcher runs from a tempdir copy with a stubbed `bun`, pinning exit 0 + all-zero sentinels when checks pass and the check's own rc (not 143) when one fails; the unit wrapper runs real two-shard fixture passes, pinning rc=0 sentinels and a real failure's rc=1. Co-authored-by: paul-0320 Co-authored-by: YMYD <53603073+OJ-OnJourney@users.noreply.github.com> Co-authored-by: Claude Fable 5 --- scripts/run-unit-parallel.sh | 13 ++- scripts/run-verify-parallel.sh | 13 ++- test/scripts/run-unit-parallel.test.ts | 92 +++++++++++++++++++- test/scripts/run-verify-parallel.test.ts | 103 ++++++++++++++++++++++- 4 files changed, 217 insertions(+), 4 deletions(-) diff --git a/scripts/run-unit-parallel.sh b/scripts/run-unit-parallel.sh index 007d5c9e1..fb6deeade 100755 --- a/scripts/run-unit-parallel.sh +++ b/scripts/run-unit-parallel.sh @@ -133,6 +133,7 @@ for i in $(seq 1 "$N"); do env SHARD="$i/$N" \ bash scripts/run-unit-shard.sh --max-concurrency="$INTRA_CONC" \ > "$SHARD_LOG" 2>&1 + rc=$? else env SHARD="$i/$N" \ bash scripts/run-unit-shard.sh --max-concurrency="$INTRA_CONC" \ @@ -142,10 +143,20 @@ for i in $(seq 1 "$N"); do sleep 5 && kill -KILL "$pid" 2>/dev/null ) & cap_pid=$! wait "$pid" 2>/dev/null + # Capture the shard's exit code from ITS `wait`, before any watchdog + # teardown runs. The teardown commands below overwrite $? — the killed + # watchdog reports 143 — which used to get stamped into every shard's + # sentinel on machines with no gtimeout/timeout: every run "failed" + # with rc=143 summaries even when all tests passed. + rc=$? + # Reap the watchdog's `sleep` child too (pkill -P), then the watchdog. + # Killing only the subshell leaves the sleep orphaned until + # $SHARD_TIMEOUT elapses — same quirk the heartbeat cleanup below works + # around; CI's orphan-process sweep flags those. + pkill -P "$cap_pid" 2>/dev/null kill "$cap_pid" 2>/dev/null wait "$cap_pid" 2>/dev/null fi - rc=$? echo "$rc" > "$LOG_DIR/shard-$i.exit" [ "$rc" = "124" ] && echo "WEDGED" > "$LOG_DIR/shard-$i.wedged" ) & diff --git a/scripts/run-verify-parallel.sh b/scripts/run-verify-parallel.sh index f03c560b9..61392801f 100755 --- a/scripts/run-verify-parallel.sh +++ b/scripts/run-verify-parallel.sh @@ -126,6 +126,7 @@ for c in "${CHECKS[@]}"; do ( if [ -n "$TIMEOUT_BIN" ]; then "$TIMEOUT_BIN" "${TIMEOUT}s" bun run "$c" > "$LOG_FILE" 2>&1 + rc=$? else bun run "$c" > "$LOG_FILE" 2>&1 & pid=$! @@ -133,10 +134,20 @@ for c in "${CHECKS[@]}"; do sleep 5 && kill -KILL "$pid" 2>/dev/null ) & cap_pid=$! wait "$pid" 2>/dev/null + # Capture the check's exit code from ITS `wait`, before any watchdog + # teardown runs. The teardown commands below overwrite $? — the killed + # watchdog reports 143 — which used to get stamped into every sentinel + # on machines with no gtimeout/timeout: verify reported pass=0 + # fail= while every per-check log said OK. + rc=$? + # Reap the watchdog's `sleep` child too (pkill -P), then the watchdog. + # Killing only the subshell leaves the sleep orphaned until $TIMEOUT + # elapses — same quirk the heartbeat cleanup in run-unit-parallel.sh + # works around; CI's orphan-process sweep flags those. + pkill -P "$cap_pid" 2>/dev/null kill "$cap_pid" 2>/dev/null wait "$cap_pid" 2>/dev/null fi - rc=$? echo "$rc" > "$EXIT_FILE" ) & PIDS+=($!) diff --git a/test/scripts/run-unit-parallel.test.ts b/test/scripts/run-unit-parallel.test.ts index e19925a2f..4227ba655 100644 --- a/test/scripts/run-unit-parallel.test.ts +++ b/test/scripts/run-unit-parallel.test.ts @@ -22,7 +22,7 @@ import { describe, it, expect, beforeAll, afterAll } from 'bun:test'; import { execFileSync, spawnSync } from 'child_process'; -import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync, rmSync, copyFileSync, chmodSync } from 'fs'; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync, rmSync, copyFileSync, chmodSync, symlinkSync } from 'fs'; import { tmpdir } from 'os'; import { join, resolve } from 'path'; @@ -154,3 +154,93 @@ describe('failing-on-purpose', () => { expect(summary).toMatch(/shard 2\/2: pass=\d+ fail=\d+ skip=\d+ rc=\d+/); }); }); + +describe('run-unit-parallel.sh no-timeout-binary fallback (rc from shard wait, not watchdog teardown)', () => { + // Forces the no-gtimeout/no-timeout branch by running the wrapper under a + // curated PATH that has every tool the scripts call EXCEPT timeout + // binaries (real `bun` symlinked in), so the fallback executes even on + // hosts with coreutils installed. + // + // Regression pinned here: the shard's sentinel .exit file must record the + // exit code read right after `wait $pid` (the shard's own rc). The + // watchdog subshell is killed with SIGTERM and reports 143; reading `$?` + // after that teardown stamped rc=143 into every shard's sentinel — the + // wrapper exited non-zero with rc=143 summaries even when every test + // passed. + let FROOT: string; + let FENV: Record; + + beforeAll(() => { + FROOT = mkdtempSync(join(tmpdir(), 'gbrain-parallel-fallback-')); + mkdirSync(join(FROOT, 'scripts'), { recursive: true }); + mkdirSync(join(FROOT, 'test'), { recursive: true }); + for (const s of ['run-unit-parallel.sh', 'run-unit-shard.sh', 'run-serial-tests.sh']) { + copyFileSync(resolve(REPO_ROOT, 'scripts', s), join(FROOT, 'scripts', s)); + chmodSync(join(FROOT, 'scripts', s), 0o755); + } + const passing = `import { describe, it, expect } from 'bun:test'; +describe('passing', () => { + it('arithmetic works', () => { expect(1 + 1).toBe(2); }); +});`; + writeFileSync(join(FROOT, 'test', 'a-pass.test.ts'), passing); + writeFileSync(join(FROOT, 'test', 'b-pass.test.ts'), passing); + + const bin = join(FROOT, 'bin'); + mkdirSync(bin); + for (const tool of ['bash', 'sh', 'env', 'dirname', 'basename', 'mktemp', 'date', 'sleep', 'cat', 'tail', 'head', 'rm', 'mkdir', 'pkill', 'grep', 'sed', 'awk', 'wc', 'tr', 'seq', 'find', 'sort', 'bun']) { + const p = Bun.which(tool); + if (p) symlinkSync(p, join(bin, tool)); + } + FENV = { + PATH: bin, + HOME: process.env.HOME ?? FROOT, + TMPDIR: process.env.TMPDIR ?? '/tmp', + GBRAIN_TEST_SHARD_TIMEOUT: '300', + }; + }); + + afterAll(() => { + if (FROOT) rmSync(FROOT, { recursive: true, force: true }); + }); + + function runFallbackWrapper(): { code: number; stdout: string; stderr: string } { + const result = spawnSync( + 'bash', + [join(FROOT, 'scripts', 'run-unit-parallel.sh'), '--shards', '2'], + { cwd: FROOT, encoding: 'utf-8', env: FENV }, + ); + return { + code: result.status ?? -1, + stdout: result.stdout || '', + stderr: result.stderr || '', + }; + } + + it('exits zero with rc=0 shard sentinels when all shards pass', () => { + const r = runFallbackWrapper(); + const summary = readFileSync(join(FROOT, '.context', 'test-summary.txt'), 'utf-8'); + expect(summary).toMatch(/shard 1\/2: pass=\d+ fail=0 skip=0 rc=0/); + expect(summary).toMatch(/shard 2\/2: pass=\d+ fail=0 skip=0 rc=0/); + expect(summary).not.toContain('rc=143'); + expect(r.code).toBe(0); + }); + + it('propagates a failing shard rc as the test runner rc (1), not the watchdog 143', () => { + const failing = `import { describe, it, expect } from 'bun:test'; +describe('failing-on-purpose', () => { + it('expects 1 to equal 2', () => { expect(1).toBe(2); }); +});`; + writeFileSync(join(FROOT, 'test', 'z-fail.test.ts'), failing); + try { + const r = runFallbackWrapper(); + expect(r.code).not.toBe(0); + const summary = readFileSync(join(FROOT, '.context', 'test-summary.txt'), 'utf-8'); + expect(summary).toMatch(/shard \d\/2: pass=\d+ fail=1 skip=0 rc=1/); + expect(summary).not.toContain('rc=143'); + const failureLog = readFileSync(join(FROOT, '.context', 'test-failures.log'), 'utf-8'); + expect(failureLog).toContain('failing-on-purpose'); + } finally { + rmSync(join(FROOT, 'test', 'z-fail.test.ts'), { force: true }); + } + }); +}); diff --git a/test/scripts/run-verify-parallel.test.ts b/test/scripts/run-verify-parallel.test.ts index 949fc3d83..8b8e64ac3 100644 --- a/test/scripts/run-verify-parallel.test.ts +++ b/test/scripts/run-verify-parallel.test.ts @@ -15,7 +15,16 @@ import { describe, expect, it } from "bun:test"; import { spawnSync } from "node:child_process"; -import { writeFileSync, mkdtempSync, rmSync } from "node:fs"; +import { + copyFileSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -173,3 +182,95 @@ exit 0 } }); }); + +describe("run-verify-parallel.sh — no-timeout-binary fallback rc capture (regression)", () => { + // macOS ships no `timeout`; without brew coreutils (`gtimeout`) — stock + // machines, minimal containers, restricted/sandboxed PATHs — the dispatcher + // degrades to the bg-pid + sleep-watchdog branch. + // + // Regression pinned here: each check's sentinel .exit file must record the + // exit code of the CHECK (read right after `wait $pid`), not of the + // watchdog teardown. The watchdog subshell is killed with SIGTERM and so + // reports 143; reading `$?` after the teardown stamped 143 into every + // sentinel — verify reported pass=0 fail= while every per-check log + // said OK. + // + // Hermetic on any host: the script runs from a tempdir copy with `bun` + // stubbed (checks complete instantly, no repo needed) and PATH set to a + // curated symlink dir containing everything the script calls EXCEPT + // gtimeout/timeout — forcing the fallback branch even where coreutils is + // installed. + + function makeFallbackHarness(): { root: string; env: Record } { + const root = mkdtempSync(join(tmpdir(), "verify-fallback-")); + mkdirSync(join(root, "scripts"), { recursive: true }); + copyFileSync(SCRIPT, join(root, "scripts", "run-verify-parallel.sh")); + + const bin = join(root, "bin"); + mkdirSync(bin); + // Everything the dispatcher and its subshells invoke, minus timeout bins. + for (const tool of ["bash", "sh", "env", "dirname", "mktemp", "date", "sleep", "cat", "tail", "head", "rm", "mkdir", "pkill", "grep", "sed", "awk"]) { + const p = Bun.which(tool); + if (p) symlinkSync(p, join(bin, tool)); + } + // `bun run ` stand-in: instant, prints OK, exits 7 for the check + // named in $STUB_FAIL_CHECK (if any). + writeFileSync( + join(bin, "bun"), + `#!/usr/bin/env bash +name="\${2:-}" +echo "stub check OK: $name" +if [ -n "\${STUB_FAIL_CHECK:-}" ] && [ "$name" = "\${STUB_FAIL_CHECK}" ]; then + echo "stub check failing: $name" >&2 + exit 7 +fi +exit 0 +`, + { mode: 0o755 }, + ); + + return { + root, + env: { + PATH: bin, + HOME: process.env.HOME ?? root, + TMPDIR: process.env.TMPDIR ?? "/tmp", + GBRAIN_VERIFY_TIMEOUT: "30", + GBRAIN_VERIFY_LOG_DIR: join(root, "logs"), + }, + }; + } + + it("all checks passing → exit 0, every sentinel records 0 (not the watchdog's 143)", () => { + const { root, env } = makeFallbackHarness(); + try { + const r = spawnSync("bash", [join(root, "scripts", "run-verify-parallel.sh")], { encoding: "utf8", env }); + expect(r.stderr).toMatch(/pass=\d+ fail=0/); + expect(r.status).toBe(0); + const exits = readdirSync(join(root, "logs")).filter((f) => f.endsWith(".exit")); + expect(exits.length).toBeGreaterThan(10); + for (const f of exits) { + expect(readFileSync(join(root, "logs", f), "utf8").trim()).toBe("0"); + } + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it("one check failing → exit 1, sentinel records the check's own rc (7), not 143", () => { + const { root, env } = makeFallbackHarness(); + try { + const r = spawnSync("bash", [join(root, "scripts", "run-verify-parallel.sh")], { + encoding: "utf8", + env: { ...env, STUB_FAIL_CHECK: "check:jsonb" }, + }); + expect(r.status).toBe(1); + expect(r.stderr).toContain("--- check:jsonb (rc=7)"); + expect(r.stderr).toContain("stub check failing: check:jsonb"); + expect(r.stderr).toMatch(/fail=1\b/); + expect(readFileSync(join(root, "logs", "check_jsonb.exit"), "utf8").trim()).toBe("7"); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +});