From da1bab532a5f2b8c98e9b40af5ff2377ad17badc Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:17:37 -0700 Subject: [PATCH 1/5] =?UTF-8?q?fix(import):=20make=20checkpoints=20staging?= =?UTF-8?q?-first=20=E2=80=94=20canonical=20dir=20identity=20+=20self-desc?= =?UTF-8?q?ribing=20metadata=20(#2935)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port of #1731 (diazMelgarejo) onto current master. gbrain import wrote ~/.gbrain/import-checkpoint.json with the caller's raw dir argument, so a checkpoint left behind by an interrupted run (e.g. SIGTERM) could carry "." or a symlinked spelling — an identity that resolves to whatever CWD the next consumer happens to run from. Downstream tooling that treated the checkpoint dir as an owned staging boundary could then act on the wrong directory. - runImport captures the import target ONCE via resolveImportTargetDir (resolve + realpathSync) and threads that canonical value through collection, checkpoint load/save, and resume filtering - checkpoints are self-describing (schema_version: 1, owner: "gbrain", kind: "import"); loadCheckpoint tolerates absent metadata (legacy path-based files) but rejects present-and-wrong metadata and any relative dir - checkpoint contract documented in docs/guides/live-sync.md (llms bundle regenerated) - test/import-resume.test.ts fixture now realpaths its tmpdir so planted checkpoints match the canonicalized dir (macOS /var -> /private/var) Fixes #1728 Co-authored-by: Sinabina Co-authored-by: Lawrence Melgarejo Co-authored-by: Claude Fable 5 --- docs/guides/live-sync.md | 10 ++++ llms-full.txt | 10 ++++ src/commands/import.ts | 18 ++++++- src/core/import-checkpoint.ts | 38 +++++++++++++- test/import-checkpoint.test.ts | 90 +++++++++++++++++++++++++++++++++- test/import-resume.test.ts | 6 ++- 6 files changed, 166 insertions(+), 6 deletions(-) diff --git a/docs/guides/live-sync.md b/docs/guides/live-sync.md index fbf0fce6c..90d3fffa3 100644 --- a/docs/guides/live-sync.md +++ b/docs/guides/live-sync.md @@ -131,6 +131,16 @@ hashes match. If both a cron and `--watch` fire simultaneously, no conflict. history rewrite still hard-blocks even with `--skip-failed`. Run `gbrain sync --skip-failed` to acknowledge a known-bad set yourself. +5. **Import checkpoints name the import target, not the caller's CWD.** + Interrupted `gbrain import ` runs may leave + `~/.gbrain/import-checkpoint.json` so the next import can resume. The + checkpoint `dir` is the absolute, resolved import target captured when + import starts. It is not a cleanup instruction and it must not be + re-derived from the process working directory. Checkpoints written by + gbrain include `schema_version: 1`, `owner: "gbrain"`, and + `kind: "import"` so downstream tools can validate the contract before + deciding whether to resume. + ## How to Verify 1. **Edit a file and search for the change.** Edit a brain markdown file, diff --git a/llms-full.txt b/llms-full.txt index 1c1dd6834..1a9710038 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -2767,6 +2767,16 @@ hashes match. If both a cron and `--watch` fire simultaneously, no conflict. history rewrite still hard-blocks even with `--skip-failed`. Run `gbrain sync --skip-failed` to acknowledge a known-bad set yourself. +5. **Import checkpoints name the import target, not the caller's CWD.** + Interrupted `gbrain import ` runs may leave + `~/.gbrain/import-checkpoint.json` so the next import can resume. The + checkpoint `dir` is the absolute, resolved import target captured when + import starts. It is not a cleanup instruction and it must not be + re-derived from the process working directory. Checkpoints written by + gbrain include `schema_version: 1`, `owner: "gbrain"`, and + `kind: "import"` so downstream tools can validate the contract before + deciding whether to resume. + ## How to Verify 1. **Edit a file and search for the change.** Edit a brain markdown file, diff --git a/src/commands/import.ts b/src/commands/import.ts index 241bcff28..1d2939318 100644 --- a/src/commands/import.ts +++ b/src/commands/import.ts @@ -20,6 +20,7 @@ import { loadCheckpoint, saveCheckpoint, clearCheckpoint, + resolveImportTargetDir, resumeFilter, } from '../core/import-checkpoint.ts'; @@ -168,7 +169,19 @@ export async function runImport( console.error('Usage: gbrain import [--no-embed] [--workers N] [--fresh] [--source-id ] [--json]'); process.exit(1); } - const dir: string = dirArg; // narrowed; survives closure capture + // #1728: capture the import target ONCE as an absolute real path. Every + // downstream consumer of `dir` (collection, checkpoint load/save, resume + // filtering) sees the same canonical identity — never the caller's `.`/ + // relative spelling, which would make the persisted checkpoint `dir` + // resolve against whatever CWD a later process happens to run from. + let dir: string; + try { + dir = resolveImportTargetDir(dirArg); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + console.error(`Import target is not readable: ${dirArg} (${msg})`); + process.exit(1); + } // v0.31.2: collect under the right strategy. Pre-fix this called // collectMarkdownFiles unconditionally — code-strategy first sync @@ -288,6 +301,9 @@ export async function runImport( catch { /* non-fatal */ } } saveCheckpoint(checkpointPath, { + schema_version: 1, + owner: 'gbrain', + kind: 'import', dir, completedPaths: Array.from(completed), timestamp: new Date().toISOString(), diff --git a/src/core/import-checkpoint.ts b/src/core/import-checkpoint.ts index 9c7cd677c..64ea03c0f 100644 --- a/src/core/import-checkpoint.ts +++ b/src/core/import-checkpoint.ts @@ -1,5 +1,5 @@ -import { existsSync, readFileSync, writeFileSync, renameSync, unlinkSync } from 'fs'; -import { relative, isAbsolute } from 'path'; +import { existsSync, readFileSync, writeFileSync, renameSync, unlinkSync, realpathSync } from 'fs'; +import { relative, isAbsolute, resolve } from 'path'; /** * Path-based import checkpoint. @@ -25,6 +25,12 @@ import { relative, isAbsolute } from 'path'; * enter the set. */ export interface ImportCheckpoint { + /** Checkpoint payload schema. v1 is path-based with explicit producer metadata. */ + schema_version: 1; + /** Producer marker for downstream consumers that validate before acting. */ + owner: 'gbrain'; + /** Checkpoint kind. Prevents unrelated checkpoint files from being treated as import state. */ + kind: 'import'; /** Absolute brain directory the checkpoint was created against. Mismatch on resume → discard. */ dir: string; /** @@ -37,6 +43,21 @@ export interface ImportCheckpoint { } const OLD_FORMAT_LOG = 'Older checkpoint format detected — re-walking (cheap via content_hash)'; +export const IMPORT_CHECKPOINT_SCHEMA_VERSION = 1; +export const IMPORT_CHECKPOINT_OWNER = 'gbrain'; +export const IMPORT_CHECKPOINT_KIND = 'import'; + +/** + * Capture the import target once at run start. `resolve()` removes caller + * spelling such as `.` or `../staging`; `realpathSync()` collapses symlinks + * and proves the target exists. The returned value is the only directory + * identity import checkpoints should ever persist (#1728 — a raw `.` here + * made the checkpoint `dir` resolve to whatever CWD the NEXT consumer ran + * from, which downstream tooling treated as an owned staging directory). + */ +export function resolveImportTargetDir(dir: string): string { + return realpathSync(resolve(dir)); +} /** * Load a checkpoint and verify it's compatible with the current run. @@ -72,11 +93,21 @@ export function loadCheckpoint(path: string, currentDir: string): ImportCheckpoi } if (typeof obj.dir !== 'string') return null; + if (!isAbsolute(obj.dir)) return null; if (obj.dir !== currentDir) return null; + // Self-describing metadata (#1728): absent fields are tolerated (legacy + // path-based checkpoints predate them), but present-and-wrong means the + // file was written by something else — don't resume from it. + if (obj.schema_version !== undefined && obj.schema_version !== IMPORT_CHECKPOINT_SCHEMA_VERSION) return null; + if (obj.owner !== undefined && obj.owner !== IMPORT_CHECKPOINT_OWNER) return null; + if (obj.kind !== undefined && obj.kind !== IMPORT_CHECKPOINT_KIND) return null; if (typeof obj.timestamp !== 'string') return null; if (!obj.completedPaths.every((p): p is string => typeof p === 'string')) return null; return { + schema_version: IMPORT_CHECKPOINT_SCHEMA_VERSION, + owner: IMPORT_CHECKPOINT_OWNER, + kind: IMPORT_CHECKPOINT_KIND, dir: obj.dir, completedPaths: obj.completedPaths, timestamp: obj.timestamp, @@ -98,6 +129,9 @@ export function saveCheckpoint(path: string, cp: ImportCheckpoint): void { // Sort for stable serialization — keeps diffs across snapshots minimal // and tests deterministic. const payload: ImportCheckpoint = { + schema_version: IMPORT_CHECKPOINT_SCHEMA_VERSION, + owner: IMPORT_CHECKPOINT_OWNER, + kind: IMPORT_CHECKPOINT_KIND, dir: cp.dir, completedPaths: [...cp.completedPaths].sort(), timestamp: cp.timestamp, diff --git a/test/import-checkpoint.test.ts b/test/import-checkpoint.test.ts index 1b6a6ad75..95208cd96 100644 --- a/test/import-checkpoint.test.ts +++ b/test/import-checkpoint.test.ts @@ -1,5 +1,5 @@ import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; -import { writeFileSync, readFileSync, existsSync, mkdtempSync, rmSync } from 'fs'; +import { writeFileSync, readFileSync, existsSync, mkdtempSync, rmSync, mkdirSync, symlinkSync, realpathSync } from 'fs'; import { tmpdir } from 'os'; import { join } from 'path'; import { @@ -7,6 +7,7 @@ import { saveCheckpoint, resumeFilter, clearCheckpoint, + resolveImportTargetDir, type ImportCheckpoint, } from '../src/core/import-checkpoint.ts'; @@ -52,6 +53,9 @@ describe('loadCheckpoint', () => { test('returns null when dir mismatches the current run', () => { const cp: ImportCheckpoint = { + schema_version: 1, + owner: 'gbrain', + kind: 'import', dir: '/other/brain', completedPaths: ['a.md'], timestamp: '2026-05-14T00:00:00Z', @@ -86,6 +90,30 @@ describe('loadCheckpoint', () => { expect(stderrCaptured).not.toContain('Older checkpoint format'); }); + test('returns null when dir is relative (#1728 — CWD-dependent identity)', () => { + writeFileSync(cpPath, JSON.stringify({ + schema_version: 1, + owner: 'gbrain', + kind: 'import', + dir: '.', + completedPaths: ['a.md'], + timestamp: '2026-01-01T00:00:00Z', + })); + expect(loadCheckpoint(cpPath, '.')).toBeNull(); + }); + + test('returns null when self-describing metadata is wrong', () => { + writeFileSync(cpPath, JSON.stringify({ + schema_version: 99, + owner: 'some-tool', + kind: 'other', + dir: '/tmp/example-brain', + completedPaths: ['a.md'], + timestamp: '2026-01-01T00:00:00Z', + })); + expect(loadCheckpoint(cpPath, '/tmp/example-brain')).toBeNull(); + }); + test('returns null when completedPaths contains non-strings', () => { writeFileSync(cpPath, JSON.stringify({ dir: '/tmp/example-brain', @@ -97,6 +125,9 @@ describe('loadCheckpoint', () => { test('returns the checkpoint for valid v0.33.2 payload', () => { const cp: ImportCheckpoint = { + schema_version: 1, + owner: 'gbrain', + kind: 'import', dir: '/tmp/example-brain', completedPaths: ['meetings/2026-05-13.md', 'concepts/foo.md'], timestamp: '2026-05-14T12:34:56Z', @@ -107,12 +138,31 @@ describe('loadCheckpoint', () => { expect(loaded?.dir).toBe('/tmp/example-brain'); expect(loaded?.completedPaths).toEqual(['meetings/2026-05-13.md', 'concepts/foo.md']); expect(loaded?.timestamp).toBe('2026-05-14T12:34:56Z'); + expect(loaded?.schema_version).toBe(1); + expect(loaded?.owner).toBe('gbrain'); + expect(loaded?.kind).toBe('import'); + }); + + test('returns legacy path-based checkpoint without metadata as v1 in memory', () => { + writeFileSync(cpPath, JSON.stringify({ + dir: '/tmp/example-brain', + completedPaths: ['a.md'], + timestamp: '2026-05-14T12:34:56Z', + })); + const loaded = loadCheckpoint(cpPath, '/tmp/example-brain'); + expect(loaded?.schema_version).toBe(1); + expect(loaded?.owner).toBe('gbrain'); + expect(loaded?.kind).toBe('import'); + expect(loaded?.dir).toBe('/tmp/example-brain'); }); }); describe('saveCheckpoint', () => { test('round-trips through loadCheckpoint', () => { const cp: ImportCheckpoint = { + schema_version: 1, + owner: 'gbrain', + kind: 'import', dir: '/tmp/example-brain', completedPaths: ['a.md', 'b.md', 'c.md'], timestamp: '2026-05-14T00:00:00Z', @@ -125,16 +175,25 @@ describe('saveCheckpoint', () => { test('serializes completedPaths sorted (deterministic output)', () => { saveCheckpoint(cpPath, { + schema_version: 1, + owner: 'gbrain', + kind: 'import', dir: '/tmp/example-brain', completedPaths: ['z.md', 'a.md', 'm.md'], timestamp: '2026-05-14T00:00:00Z', }); const onDisk = JSON.parse(readFileSync(cpPath, 'utf-8')); + expect(onDisk.schema_version).toBe(1); + expect(onDisk.owner).toBe('gbrain'); + expect(onDisk.kind).toBe('import'); expect(onDisk.completedPaths).toEqual(['a.md', 'm.md', 'z.md']); }); test('atomic-ish write — no stray .tmp file after success', () => { saveCheckpoint(cpPath, { + schema_version: 1, + owner: 'gbrain', + kind: 'import', dir: '/tmp/example-brain', completedPaths: ['a.md'], timestamp: '2026-05-14T00:00:00Z', @@ -148,6 +207,9 @@ describe('saveCheckpoint', () => { const badPath = join(workDir, 'does-not-exist', 'cp.json'); expect(() => saveCheckpoint(badPath, { + schema_version: 1, + owner: 'gbrain', + kind: 'import', dir: '/tmp/example-brain', completedPaths: ['a.md'], timestamp: '2026-05-14T00:00:00Z', @@ -157,6 +219,32 @@ describe('saveCheckpoint', () => { }); }); +describe('resolveImportTargetDir', () => { + test('captures a relative import target as an absolute real path', () => { + const target = join(workDir, 'staging'); + mkdirSync(target); + const cwd = process.cwd(); + try { + process.chdir(workDir); + expect(resolveImportTargetDir('staging')).toBe(realpathSync(target)); + } finally { + process.chdir(cwd); + } + }); + + test('collapses symlink spelling to the real import target', () => { + const target = join(workDir, 'real-staging'); + const link = join(workDir, 'linked-staging'); + mkdirSync(target); + symlinkSync(target, link); + expect(resolveImportTargetDir(link)).toBe(realpathSync(target)); + }); + + test('throws when the target does not exist', () => { + expect(() => resolveImportTargetDir(join(workDir, 'nope'))).toThrow(); + }); +}); + describe('resumeFilter', () => { test('empty completed set returns all files unchanged', () => { const all = ['a.md', 'b.md', 'c.md']; diff --git a/test/import-resume.test.ts b/test/import-resume.test.ts index e266ba45b..278f62b4e 100644 --- a/test/import-resume.test.ts +++ b/test/import-resume.test.ts @@ -20,7 +20,7 @@ * `afterAll`) per CLAUDE.md test-isolation rules R3 + R4. */ import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach } from 'bun:test'; -import { mkdtempSync, writeFileSync, readFileSync, existsSync, rmSync, mkdirSync } from 'fs'; +import { mkdtempSync, writeFileSync, readFileSync, existsSync, rmSync, mkdirSync, realpathSync } from 'fs'; import { tmpdir } from 'os'; import { join } from 'path'; import { PGLiteEngine } from '../src/core/pglite-engine.ts'; @@ -52,7 +52,9 @@ beforeEach(async () => { gbrainHomeDir = join(workspace, '.gbrain'); mkdirSync(gbrainHomeDir, { recursive: true }); cpPath = join(gbrainHomeDir, 'import-checkpoint.json'); - brainDir = mkdtempSync(join(tmpdir(), 'gbrain-import-resume-brain-')); + // #1728: realpath so planted checkpoints match runImport's canonicalized + // dir (macOS tmpdir is a /var → /private/var symlink). + brainDir = realpathSync(mkdtempSync(join(tmpdir(), 'gbrain-import-resume-brain-'))); }); afterEach(() => { From 2df41a84c9cbca6bea4c6d59a6f93ec9247cfa02 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:18:48 -0700 Subject: [PATCH 2/5] feat(ai-gateway): derive OpenAI prompt_cache_key for native-OpenAI chat models (takeover of #2442) (#2933) Ports the still-unmerged half of PR #2442. OpenAI caches prompt prefixes automatically, but a stable prompt_cache_key keeps requests that share a prefix on the same inference engine, lifting the automatic-cache hit rate. chat() now derives a stable key from the system prompt + sorted tool names for native-openai models and passes it via providerOptions.openai. promptCacheKey. Config provider_chat_options still overrides the derived key; anthropic/google/openai-compatible providers are untouched. The Anthropic half of #2442 (cache_control "silent no-op") is superseded: @ai-sdk/anthropic 3.0.74 forwards call-level providerOptions.anthropic. cacheControl as the Messages API's request-level cache_control (automatic prefix caching), so master's existing marker is live on current deps. Co-authored-by: Sinabina Co-authored-by: CoachRyanNguyen Co-authored-by: Claude Fable 5 --- src/core/ai/gateway.ts | 39 ++++++ .../gateway-openai-prompt-cache-key.test.ts | 123 ++++++++++++++++++ 2 files changed, 162 insertions(+) create mode 100644 test/ai/gateway-openai-prompt-cache-key.test.ts diff --git a/src/core/ai/gateway.ts b/src/core/ai/gateway.ts index 8a169e5fd..789c2db1e 100644 --- a/src/core/ai/gateway.ts +++ b/src/core/ai/gateway.ts @@ -23,6 +23,7 @@ import { embed as aiEmbed, embedMany, generateObject, generateText, jsonSchema } from 'ai'; import { AsyncLocalStorage } from 'node:async_hooks'; +import { createHash } from 'node:crypto'; import { listRecipes } from './recipes/index.ts'; import { createOpenAI } from '@ai-sdk/openai'; import { createGoogleGenerativeAI } from '@ai-sdk/google'; @@ -2849,6 +2850,30 @@ async function classifyGatewayGuardrail(input: { } } +/** + * Derive OpenAI's `prompt_cache_key` (the AI SDK's `providerOptions.openai. + * promptCacheKey`). It's a ROUTING hint, not a cache breakpoint: OpenAI caches + * prefixes automatically, and a stable key makes requests sharing a prefix + * land on the same engine, raising the hit rate (OpenAI cites 60%→87%). + * + * Hash the system prompt + sorted tool names — that's the stable prefix + * gbrain's repeated loops (enrich, page-summary, skillopt, subagent) actually + * share. Returns undefined when there's no system prompt (nothing stable to + * key on), so one-off requests don't get pinned to a single engine. An + * explicit key can still be set per provider/model via + * `provider_chat_options` config, which overrides the derived key. + * + * @internal exported for tests; not part of the public gateway API. + */ +export function openAIPromptCacheKey(args: { + system?: string; + toolNames?: string[]; +}): string | undefined { + if (!args.system) return undefined; + const basis = `${args.system} ${(args.toolNames ?? []).slice().sort().join(',')}`; + return `gbrain:${createHash('sha256').update(basis).digest('hex').slice(0, 32)}`; +} + export function toAISDKTools(tools: ChatToolDef[] | undefined): Record | undefined { if (!tools || tools.length === 0) return undefined; return tools.reduce((acc, t) => { @@ -2959,6 +2984,20 @@ export async function chat(opts: ChatOpts): Promise { if (useCache) { providerOptions.anthropic = { cacheControl: { type: 'ephemeral' } }; } + // OpenAI prompt_cache_key (native-openai only): a stable per-prefix routing + // hint that keeps requests sharing a system prompt + tool set on the same + // inference engine, lifting OpenAI's automatic prefix-cache hit rate. The + // openai-compatible path (litellm/azure/groq/...) ignores + // providerOptions.openai, so it gets nothing. Applied BEFORE the configured + // provider options so `provider_chat_options.openai.promptCacheKey` from + // config still overrides the derived key. + if (recipe.implementation === 'native-openai') { + const promptCacheKey = openAIPromptCacheKey({ + system: opts.system, + toolNames: (opts.tools ?? []).map(t => t.name), + }); + if (promptCacheKey) providerOptions.openai = { promptCacheKey }; + } applyConfiguredChatProviderOptions(providerOptions, cfg, recipe.id, modelId); let _budgetRecorded = false; diff --git a/test/ai/gateway-openai-prompt-cache-key.test.ts b/test/ai/gateway-openai-prompt-cache-key.test.ts new file mode 100644 index 000000000..8d49e56b2 --- /dev/null +++ b/test/ai/gateway-openai-prompt-cache-key.test.ts @@ -0,0 +1,123 @@ +/** + * OpenAI `prompt_cache_key` routing hint (takeover of PR #2442's remaining + * half, originally by @CoachRyanNguyen). + * + * OpenAI caches prompt prefixes automatically; a stable `prompt_cache_key` + * keeps requests that share a prefix on the same inference engine, lifting the + * automatic-cache hit rate. `chat()` derives one from the system prompt + tool + * names for native-OpenAI models and passes it via + * `providerOptions.openai.promptCacheKey` (which @ai-sdk/openai maps to the + * request's `prompt_cache_key`). + * + * Pins: + * - key derivation is stable (tool ORDER doesn't matter), sensitive to + * system/tool-set changes, and absent without a system prompt + * - the chat() wiring only fires for native-openai (anthropic/compat get + * nothing), and config `provider_chat_options` overrides the derived key + */ + +import { describe, test, expect, beforeEach } from 'bun:test'; +import { + chat, + configureGateway, + openAIPromptCacheKey, + resetGateway, + __setGenerateTextTransportForTests, +} from '../../src/core/ai/gateway.ts'; + +describe('openAIPromptCacheKey — derivation', () => { + test('same system + same tools → identical stable key (sticky routing)', () => { + const a = openAIPromptCacheKey({ system: 'SYS', toolNames: ['search', 'put_page'] }); + const b = openAIPromptCacheKey({ system: 'SYS', toolNames: ['put_page', 'search'] }); + expect(a).toBe(b as string); // tool ORDER must not change the key + expect(a).toMatch(/^gbrain:[0-9a-f]{32}$/); + }); + + test('different system → different key', () => { + const a = openAIPromptCacheKey({ system: 'SYS A', toolNames: [] }); + const b = openAIPromptCacheKey({ system: 'SYS B', toolNames: [] }); + expect(a).not.toBe(b as string); + }); + + test('different tool set → different key', () => { + const a = openAIPromptCacheKey({ system: 'SYS', toolNames: ['search'] }); + const b = openAIPromptCacheKey({ system: 'SYS', toolNames: ['search', 'put_page'] }); + expect(a).not.toBe(b as string); + }); + + test('no system prompt → undefined (do not pin one-off requests)', () => { + expect(openAIPromptCacheKey({ system: undefined, toolNames: ['search'] })).toBeUndefined(); + }); +}); + +describe('chat() wiring — prompt_cache_key per provider', () => { + beforeEach(() => { + resetGateway(); + __setGenerateTextTransportForTests(null); + }); + + async function captureProviderOptions( + config: Parameters[0], + opts: Partial[0]> = {}, + ): Promise | undefined> { + let captured: Record | undefined; + __setGenerateTextTransportForTests(async (args: any) => { + captured = args.providerOptions; + return { + content: [{ type: 'text', text: 'ok' }], + finishReason: 'stop', + usage: { inputTokens: 1, outputTokens: 1 }, + } as any; + }); + configureGateway(config); + await chat({ + model: config.chat_model ?? 'anthropic:claude-sonnet-4-6', + messages: [{ role: 'user', content: 'hello' }], + ...opts, + }); + return captured; + } + + test('native-openai with a system prompt → providerOptions.openai.promptCacheKey', async () => { + const providerOptions = await captureProviderOptions( + { chat_model: 'openai:gpt-4o-mini', env: { OPENAI_API_KEY: 'fake' } }, + { system: 'SYS' }, + ); + expect(providerOptions?.openai?.promptCacheKey).toMatch(/^gbrain:[0-9a-f]{32}$/); + }); + + test('native-openai without a system prompt → no providerOptions at all', async () => { + const providerOptions = await captureProviderOptions( + { chat_model: 'openai:gpt-4o-mini', env: { OPENAI_API_KEY: 'fake' } }, + ); + expect(providerOptions).toBeUndefined(); + }); + + test('native-anthropic never gets an openai promptCacheKey', async () => { + const providerOptions = await captureProviderOptions( + { chat_model: 'anthropic:claude-sonnet-4-6', env: { ANTHROPIC_API_KEY: 'fake' } }, + { system: 'SYS' }, + ); + expect(providerOptions?.openai).toBeUndefined(); + }); + + test('openai-compatible (deepseek) never gets promptCacheKey (provider ignores providerOptions.openai)', async () => { + const providerOptions = await captureProviderOptions( + { chat_model: 'deepseek:deepseek-chat', env: { DEEPSEEK_API_KEY: 'fake' } }, + { system: 'SYS' }, + ); + expect(providerOptions?.openai).toBeUndefined(); + }); + + test('config provider_chat_options overrides the derived key', async () => { + const providerOptions = await captureProviderOptions( + { + chat_model: 'openai:gpt-4o-mini', + env: { OPENAI_API_KEY: 'fake' }, + provider_chat_options: { openai: { promptCacheKey: 'session-42' } }, + }, + { system: 'SYS' }, + ); + expect(providerOptions?.openai?.promptCacheKey).toBe('session-42'); + }); +}); From bd2ba46a616855eee5a1a2399c8d2ba9f67144da Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:18:52 -0700 Subject: [PATCH 3/5] fix(retry): reconnect on null instance pool in ALL non-batch config accessors (takeover of #1891) (#2934) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports the still-unmerged remnant of PR #1891 (#1593 follow-up). Master's getConfig gained retry-with-reconnect in #1603, but its siblings — setConfig, unsetConfig, listConfigKeys — still touched `this.sql` bare, so the first config write/list after a mid-cycle instance-pool teardown threw the retryable "No database connection" (issue #1678) unhandled instead of rebuilding the pool. Adds the connRetry helper from #1891 (same retry+reconnect posture as batchRetry, but no batch audit JSONL — a config accessor is not a sized batch), refactors getConfig onto it, and wraps the other three. Writes are safe to retry: withRetry only retries connection-class failures and both writes are idempotent (upsert / delete). Co-authored-by: Sinabina Co-authored-by: jalagrange Co-authored-by: Claude Fable 5 --- src/core/postgres-engine.ts | 92 ++++++++++++------- test/postgres-engine-config-reconnect.test.ts | 86 +++++++++++++++++ 2 files changed, 146 insertions(+), 32 deletions(-) create mode 100644 test/postgres-engine-config-reconnect.test.ts diff --git a/src/core/postgres-engine.ts b/src/core/postgres-engine.ts index 817a7c1b7..2fe57e517 100644 --- a/src/core/postgres-engine.ts +++ b/src/core/postgres-engine.ts @@ -5244,52 +5244,80 @@ export class PostgresEngine implements BrainEngine { } // Config + + /** + * Single-statement sibling of {@link batchRetry} for the NON-batch config + * accessors that touch `this.sql` directly (#1603 / PR #1593 follow-up, + * PR #1891 by @jalagrange). + * + * Why not `batchRetry`: a config accessor is not a sized batch — routing it + * through `batchRetry` would emit bogus batch-retry audit JSONL (inflating + * the `batch_retry_health` doctor metric) and demand a fake `BatchAuditSite` + * enum member. This keeps the SAME retry + reconnect posture with no audit. + * + * Why it exists: the `sql` getter throws a RETRYABLE "No database + * connection" by design when an instance pool was torn down mid-cycle + * (#1678), precisely so a withRetry+reconnect caller rebuilds the pool and + * self-heals. `getConfig` got that wrapper in #1603; the sibling accessors + * did not — so the first config write/list after a mid-cycle disconnect + * threw unhandled (e.g. crashing the worker into a respawn loop). + * + * `fn` MUST re-read `this.sql` per invocation — `reconnect()` rebuilds the + * pool between attempts. Safe for the writes too: `withRetry` only retries + * connection-class failures (statement never committed), and both writes + * are idempotent (upsert / delete), so even a lost-ack replay converges. + */ + private async connRetry(fn: () => Promise): Promise { + const opts = this.getBulkRetryOpts(); + return withRetry(fn, { + maxRetries: opts.maxRetries, + delayMs: opts.delayMs, + delayMaxMs: opts.delayMaxMs, + jitter: BULK_RETRY_OPTS.jitter, + // Same reconnect posture as batchRetry: rebuild a dead instance pool + // before the next attempt. Race-safe via the engine's `_reconnecting` + // guard; fail-loud — a reconnect throw propagates as the real cause. + reconnect: (ctx) => this.reconnect(ctx), + }); + } + async getConfig(key: string): Promise { // #1603: a transient pooler drop on this read used to throw / fall through // to defaults silently — which on remote Postgres surfaces as the wrong - // search mode/knobs and empty-stdout queries. Retry-with-reconnect using the - // same tuned opts as the bulk writers. No auditSite: this is a single-row - // read, not a bulk write, so it must not emit batch-retry audit rows. - // `this.sql` is a getter, so each attempt sees the pool rebuilt by reconnect. - const opts = this.getBulkRetryOpts(); - return withRetry( - async () => { - const rows = await this.sql`SELECT value FROM config WHERE key = ${key}`; - return rows.length > 0 ? (rows[0].value as string) : null; - }, - { - maxRetries: opts.maxRetries, - delayMs: opts.delayMs, - delayMaxMs: opts.delayMaxMs, - jitter: BULK_RETRY_OPTS.jitter, - reconnect: (ctx) => this.reconnect(ctx), - }, - ); + // search mode/knobs and empty-stdout queries. + return this.connRetry(async () => { + const rows = await this.sql`SELECT value FROM config WHERE key = ${key}`; + return rows.length > 0 ? (rows[0].value as string) : null; + }); } async setConfig(key: string, value: string): Promise { - const sql = this.sql; - await sql` - INSERT INTO config (key, value) VALUES (${key}, ${value}) - ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value - `; + return this.connRetry(async () => { + await this.sql` + INSERT INTO config (key, value) VALUES (${key}, ${value}) + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value + `; + }); } async unsetConfig(key: string): Promise { - const sql = this.sql; - const result = await sql`DELETE FROM config WHERE key = ${key}` as unknown as { count: number }; - return result.count ?? 0; + return this.connRetry(async () => { + const result = await this.sql`DELETE FROM config WHERE key = ${key}` as unknown as { count: number }; + return result.count ?? 0; + }); } async listConfigKeys(prefix: string): Promise { - const sql = this.sql; - // LIKE-escape literal % and _ so a config key with those chars resolves correctly. + // LIKE-escape literal % and _ so a config key with those chars resolves + // correctly. Pure string work — stays outside the retried thunk. const escaped = prefix.replace(/\\/g, '\\\\').replace(/%/g, '\\%').replace(/_/g, '\\_'); const pattern = `${escaped}%`; - const rows = await sql<{ key: string }[]>` - SELECT key FROM config WHERE key LIKE ${pattern} ESCAPE '\\' ORDER BY key - `; - return rows.map(r => r.key); + return this.connRetry(async () => { + const rows = await this.sql<{ key: string }[]>` + SELECT key FROM config WHERE key LIKE ${pattern} ESCAPE '\\' ORDER BY key + `; + return rows.map(r => r.key); + }); } // Migration support diff --git a/test/postgres-engine-config-reconnect.test.ts b/test/postgres-engine-config-reconnect.test.ts new file mode 100644 index 000000000..df44d47c9 --- /dev/null +++ b/test/postgres-engine-config-reconnect.test.ts @@ -0,0 +1,86 @@ +/** + * Non-batch config accessors must self-heal the same way the batch path does + * (takeover of PR #1891 by @jalagrange; #1593 follow-up). + * + * The config accessors touch `this.sql` directly. When an instance pool is + * torn down mid-cycle the getter throws a RETRYABLE "No database connection" + * (issue #1678) by design, so a withRetry+reconnect caller can rebuild the + * pool and recover. `getConfig` got that wrapper in #1603; `setConfig`, + * `unsetConfig`, and `listConfigKeys` did not — so the first config write or + * list after a mid-cycle disconnect threw unhandled. This pins that ALL four + * accessors now reconnect + retry, and that non-retryable errors are NOT + * masked by a reconnect. + * + * Pure: pokes private fields and stubs `reconnect` to simulate the pool + * rebuild; no real DB. + */ + +import { describe, it, expect } from 'bun:test'; +import { PostgresEngine } from '../src/core/postgres-engine.ts'; + +// A tagged-template-callable fake `sql` that resolves to the given value. +function fakeSql(result: unknown) { + return (..._args: unknown[]) => Promise.resolve(result); +} + +// Near-instant retry delays so the inter-attempt sleep does not slow the test. +// Shape matches `resolveBulkRetryOpts()` (the getBulkRetryOpts cache type). +const FAST_RETRY = { maxRetries: 3, delayMs: 1, delayMaxMs: 1, jitter: 'none' as const }; + +/** Engine with a torn-down instance pool; reconnect installs `poolResult`. */ +function makeTornDownEngine(poolResult: unknown): { engine: PostgresEngine; reconnects: () => number } { + const e = new PostgresEngine(); + (e as unknown as { _connectionStyle: string })._connectionStyle = 'instance'; + (e as unknown as { _sql: unknown })._sql = null; // instance pool torn down → getter throws retryable + (e as unknown as { _bulkRetryOptsCache: unknown })._bulkRetryOptsCache = FAST_RETRY; + let reconnectCalls = 0; + (e as unknown as { reconnect: () => Promise }).reconnect = async () => { + reconnectCalls++; + (e as unknown as { _sql: unknown })._sql = fakeSql(poolResult); + }; + return { engine: e, reconnects: () => reconnectCalls }; +} + +describe('PostgresEngine non-batch config accessors self-heal (PR #1891 takeover)', () => { + it('getConfig reconnects + retries a null instance pool, then returns the value', async () => { + const { engine, reconnects } = makeTornDownEngine([{ value: 'live-value' }]); + expect(await engine.getConfig('some.key')).toBe('live-value'); + expect(reconnects()).toBe(1); // exactly one reconnect closed the gap + }); + + it('setConfig reconnects + retries a null instance pool (idempotent upsert)', async () => { + const { engine, reconnects } = makeTornDownEngine([]); + await engine.setConfig('some.key', 'v'); + expect(reconnects()).toBe(1); + }); + + it('unsetConfig reconnects + retries a null instance pool, then returns the count', async () => { + const { engine, reconnects } = makeTornDownEngine({ count: 2 }); + expect(await engine.unsetConfig('some.key')).toBe(2); + expect(reconnects()).toBe(1); + }); + + it('listConfigKeys reconnects + retries a null instance pool, then returns keys', async () => { + const { engine, reconnects } = makeTornDownEngine([{ key: 'a.one' }, { key: 'a.two' }]); + expect(await engine.listConfigKeys('a.')).toEqual(['a.one', 'a.two']); + expect(reconnects()).toBe(1); + }); + + it('surfaces a non-retryable error without reconnecting (no masking)', async () => { + const e = new PostgresEngine(); + (e as unknown as { _connectionStyle: string })._connectionStyle = 'instance'; + (e as unknown as { _bulkRetryOptsCache: unknown })._bulkRetryOptsCache = FAST_RETRY; + // A live pool whose query throws a NON-retryable (non-connection) error. + (e as unknown as { _sql: unknown })._sql = () => + Promise.reject(new Error('syntax error at or near "SLECT"')); + + let reconnectCalls = 0; + (e as unknown as { reconnect: () => Promise }).reconnect = async () => { + reconnectCalls++; + }; + + await expect(e.getConfig('k')).rejects.toThrow('syntax error'); + await expect(e.setConfig('k', 'v')).rejects.toThrow('syntax error'); + expect(reconnectCalls).toBe(0); // non-retryable → no reconnect, error not masked + }); +}); From a0ef9515867945c5cb5dfd13ccb35cb37c1e8081 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:18:56 -0700 Subject: [PATCH 4/5] fix(dream): gate patterns phase on gateway provider reachability, not ANTHROPIC_API_KEY (takeover of #2279) (#2936) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Absorbs PR #2279's intent (drop the hardcoded ANTHROPIC_API_KEY gate) with the end-state the gateway world actually wants: the patterns phase now probes the RESOLVED patterns model through probeChatModel(normalizeModelId) — the same semantics as think/index.ts and synthesize's makeJudgeClient. Fixes two misclassifications of the old env gate: - Non-Anthropic stacks (litellm, deepseek, openrouter, ...) were skipped as "no upstream" even though the subagent routes them through the gateway (agent.use_gateway_loop). They now pass the gate; their auth is checked lazily at dispatch and surfaces in the job outcome. - Anthropic keys set via `gbrain config set anthropic_api_key` (stdio MCP launches without shell env) were treated as missing. hasAnthropicKey inside probeChatModel reads both sources. Skip reason renames no_api_key → no_provider (carrying the probe's detail). Both pinning tests updated: the structural test asserts the probe wiring; the PGLite E2E swaps its env-only helper for the shared hermetic withoutAnthropicKey (env + config file) so it can't flip to a live LLM call on a dev machine with a config-file key. Co-authored-by: Sinabina Co-authored-by: brettdavies Co-authored-by: Claude Fable 5 --- src/core/cycle/patterns.ts | 19 +++++++++++++++--- test/cycle-patterns.test.ts | 11 ++++++++--- test/e2e/dream-patterns-pglite.test.ts | 27 ++++++++++++-------------- 3 files changed, 36 insertions(+), 21 deletions(-) diff --git a/src/core/cycle/patterns.ts b/src/core/cycle/patterns.ts index 75e798ea4..5edb28f7e 100644 --- a/src/core/cycle/patterns.ts +++ b/src/core/cycle/patterns.ts @@ -27,6 +27,8 @@ import { waitForCompletion, TimeoutError } from '../minions/wait-for-completion. import type { MinionJobInput, SubagentHandlerData } from '../minions/types.ts'; import { serializeMarkdown } from '../markdown.ts'; import type { Page, PageType } from '../types.ts'; +import { probeChatModel } from '../ai/gateway.ts'; +import { normalizeModelId } from '../model-id.ts'; export interface PatternsPhaseOpts { brainDir: string; @@ -63,9 +65,20 @@ export async function runPhasePatterns( }); } - // Submit one subagent for pattern detection. - if (!process.env.ANTHROPIC_API_KEY) { - return skipped('no_api_key', 'ANTHROPIC_API_KEY unset; pattern detection skipped'); + // Submit one subagent for pattern detection. The subagent dispatches via + // the gateway model-tier resolver, so gate on "is the resolved model's + // provider reachable" rather than ANTHROPIC_API_KEY specifically — a + // hardcoded env gate misclassified non-Anthropic stacks (litellm, + // deepseek, openrouter, ...) as "no upstream" even though the subagent + // routes them through the gateway (agent.use_gateway_loop), and it missed + // Anthropic keys set via `gbrain config set anthropic_api_key`. Same + // probe semantics as think/index.ts + synthesize's makeJudgeClient: + // unknown provider/model or Anthropic-without-key skips cheaply; other + // providers' auth is checked lazily at dispatch and surfaces in the job + // outcome. (Takeover of PR #2279's intent by @brettdavies.) + const probe = probeChatModel(normalizeModelId(config.model)); + if (!probe.ok) { + return skipped('no_provider', `pattern detection skipped: ${probe.detail}`); } const allowedSlugPrefixes = await loadAllowedSlugPrefixes(); diff --git a/test/cycle-patterns.test.ts b/test/cycle-patterns.test.ts index 4e8753132..1ea368d89 100644 --- a/test/cycle-patterns.test.ts +++ b/test/cycle-patterns.test.ts @@ -39,9 +39,14 @@ describe('patterns phase wiring', () => { expect(patternsSrc).toContain("tool_name = 'brain_put_page'"); }); - test('skips when ANTHROPIC_API_KEY missing', () => { - expect(patternsSrc).toContain('ANTHROPIC_API_KEY'); - expect(patternsSrc).toContain('no_api_key'); + test('gates on gateway provider reachability, not ANTHROPIC_API_KEY (PR #2279)', () => { + // The gate must probe the RESOLVED patterns model through the gateway + // (any configured provider can run patterns), not hardcode the Anthropic + // env var — that misclassified non-Anthropic stacks as "no upstream". + expect(patternsSrc).toContain('probeChatModel'); + expect(patternsSrc).toContain('normalizeModelId'); + expect(patternsSrc).toContain('no_provider'); + expect(patternsSrc).not.toContain('process.env.ANTHROPIC_API_KEY'); }); test('skips when reflections below min_evidence', () => { diff --git a/test/e2e/dream-patterns-pglite.test.ts b/test/e2e/dream-patterns-pglite.test.ts index 88474f791..8d55f4a5e 100644 --- a/test/e2e/dream-patterns-pglite.test.ts +++ b/test/e2e/dream-patterns-pglite.test.ts @@ -9,7 +9,9 @@ * Anthropic call: * - disabled: dream.patterns.enabled=false → skipped * - insufficient_evidence: { }; } -async function withoutAnthropicKey(body: () => Promise): Promise { - const saved = process.env.ANTHROPIC_API_KEY; - delete process.env.ANTHROPIC_API_KEY; - try { - return await body(); - } finally { - if (saved === undefined) delete process.env.ANTHROPIC_API_KEY; - else process.env.ANTHROPIC_API_KEY = saved; - } -} - /** * Insert N reflection pages directly via engine.putPage so the patterns * gather query has data without going through the synthesize phase. @@ -141,18 +133,23 @@ describe('E2E patterns — insufficient_evidence', () => { }, 30_000); }); -describe('E2E patterns — no API key', () => { - test('enough reflections, no ANTHROPIC_API_KEY → skipped no_api_key', async () => { +describe('E2E patterns — no reachable provider', () => { + test('enough reflections, no Anthropic key in env OR config → skipped no_provider', async () => { const rig = await setupRig(); try { await seedReflections(rig.engine, 5); // above default min_evidence (3) + // Default patterns model resolves to Anthropic; with no key reachable + // from EITHER source (env + config file — the shared helper neuters + // both) the gateway probe reports the provider unavailable. A + // non-Anthropic stack (litellm, deepseek, ...) passes this gate and + // dispatches through the gateway instead (PR #2279). await withoutAnthropicKey(async () => { const result = await runPhasePatterns(rig.engine, { brainDir: rig.brainDir, dryRun: false, }); expect(result.status).toBe('skipped'); - expect((result.details as { reason?: string }).reason).toBe('no_api_key'); + expect((result.details as { reason?: string }).reason).toBe('no_provider'); }); } finally { await rig.cleanup(); From e1cefd065402c069db72f89c01c08c78c78fb3f3 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:25:48 -0700 Subject: [PATCH 5/5] =?UTF-8?q?fix(subagent):=20orchestration=20fix-wave?= =?UTF-8?q?=20G=20=E2=80=94=20configurable=20timeouts/caps,=20honest=20chi?= =?UTF-8?q?ld=20outcomes,=20fenced=20timeline=20writes=20(#2937)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four verified-open issues in the subagent-orchestration family, one PR: - #1594: dream synthesize subagent job/wait timeouts promoted from hardcoded 30/35-min constants to config keys dream.synthesize.subagent_timeout_ms / subagent_wait_timeout_ms. Approach ported from stale PR #1596 (credit @ai920wisco). - #2778: add_timeline_entry joins the subagent brain-tool allowlist, fenced fail-closed by the shared enforceSubagentSlugFence (extracted from put_page's inline check — same trusted-workspace allow-list / wiki/agents// namespace policy). The per-turn output cap is now resolveMaxOutputTokens (data.max_tokens → agent.max_output_tokens → 8192, was hardcoded 4096); a max_tokens stop surfaces as stop_reason 'max_tokens' instead of a silent end_turn, and a mid-tool-round cap hit injects a truncation note so the model re-issues the dropped call. - #2782: patterns phase status now reflects the child outcome — non-complete outcome with zero writes → fail (PATTERNS_CHILD_), partial writes → warn. Patterns timeouts get the same config-key pair (dream.patterns.subagent_timeout_ms / subagent_wait_timeout_ms). - #2113: facts extraction cap is config facts.extraction_max_tokens (default 4000, was hardcoded 1500); stopReason 'length' is checked, retried once at 2x the cap, and surfaced on stderr instead of silently extracting zero facts. Co-authored-by: Sinabina Co-authored-by: ai920wisco Co-authored-by: Claude Fable 5 --- docs/architecture/KEY_FILES.md | 12 +- src/core/config.ts | 26 ++++ src/core/cycle/patterns.ts | 62 +++++++- src/core/cycle/synthesize.ts | 31 +++- src/core/facts/extract.ts | 63 +++++++-- src/core/minions/handlers/subagent.ts | 57 +++++++- src/core/minions/tools/brain-allowlist.ts | 7 + src/core/minions/types.ts | 7 + src/core/operations.ts | 72 ++++++---- test/brain-allowlist.serial.test.ts | 5 +- test/config-set.test.ts | 5 + test/cycle-patterns-child-outcome.test.ts | 103 ++++++++++++++ .../cycle-synthesize-subagent-timeout.test.ts | 83 +++++++++++ test/facts-extract-truncation.test.ts | 132 ++++++++++++++++++ test/loadConfig-merge.test.ts | 6 +- test/subagent-handler.test.ts | 112 +++++++++++++++ test/timeline-entry-subagent-fence.test.ts | 102 ++++++++++++++ 17 files changed, 825 insertions(+), 60 deletions(-) create mode 100644 test/cycle-patterns-child-outcome.test.ts create mode 100644 test/cycle-synthesize-subagent-timeout.test.ts create mode 100644 test/facts-extract-truncation.test.ts create mode 100644 test/timeline-entry-subagent-fence.test.ts diff --git a/docs/architecture/KEY_FILES.md b/docs/architecture/KEY_FILES.md index 5484e8c21..b40e6a45f 100644 --- a/docs/architecture/KEY_FILES.md +++ b/docs/architecture/KEY_FILES.md @@ -8,7 +8,7 @@ lives in `CHANGELOG.md` + `git log` / `git blame`, NOT here. Do not append per-release `**vX.Y.Z:**` narration — CI enforces this (`scripts/check-key-files-current-state.sh`). -- `src/core/operations.ts` — Contract-first operation definitions (the foundation). Exports upload validators `validateUploadPath`, `validatePageSlug`, `validateFilename`, plus `matchesSlugAllowList(slug, prefixes)` (glob matcher: `/*` matches recursive children; bare `` matches exact only). `OperationContext.remote` is a REQUIRED field flagging untrusted callers; `OperationContext.allowedSlugPrefixes` is the trusted-workspace allow-list set by the dream cycle; `OperationContext.auth?: AuthInfo` is threaded through HTTP dispatch for scope enforcement in `serve-http.ts` before the op runs. `put_page` enforces: when `viaSubagent` and `allowedSlugPrefixes` is set, slug must match the allow-list; else the legacy `wiki/agents//...` namespace check applies. Auto-link skipped only when `remote=true && !trustedWorkspace`. Every `Operation` carries `scope?: 'read' | 'write' | 'admin'` + `localOnly?: boolean`; `sync_brain`, `file_upload`, `file_list`, `file_url` are `admin + localOnly` (rejected over HTTP). Four trust-boundary call sites (`put_page` allowlist, file_upload trust-narrowing, submit_job protected-name guard, auto-link skip) use FAIL-CLOSED semantics: `ctx.remote === false` for trusted-only sites, `ctx.remote !== false` for "untrust unless explicit-false" — anything not strictly `false` is treated as remote (closes the HTTP MCP shell-job RCE where a read+write OAuth token could submit `shell` jobs). `sourceScopeOpts(ctx)` encodes the source-scoped read precedence ladder — federated array (`ctx.auth.allowedSources`) wins over scalar (`ctx.sourceId`/`ctx.auth.sourceId`) over nothing; every read-side op handler routes through it so a source-bound OAuth client can't see neighboring sources via `search`/`query`/`list_pages`/`get_page`/`find_experts`/query's image path, plus the by-slug reads `get_tags`/`get_links`/`get_backlinks`/`get_timeline` (and `get_page`'s tag fetch, which resolves against the concrete page's own `source_id`). `linkReadScopeOpts(ctx)` is the link-read sibling for `get_links`/`get_backlinks`: a link row references three pages (from/to/origin), and the engine's federated (`sourceIds[]`) branch scopes ALL THREE while its scalar (`sourceId`) branch scopes only the near endpoint (by design — trusted internal callers like `reconcileLinks` and back-link validators call the engine with a scalar scope and need the cross-source view). For an UNTRUSTED remote caller (`ctx.remote !== false`) carrying only a scalar scope it promotes that scope to a single-element `sourceIds:[id]`, routing them through the all-endpoint branch so a far/origin slug from outside the grant isn't disclosed; a federated array passes through unchanged and trusted local CLI (`ctx.remote === false`) keeps the scalar cross-source view. `thinkSourceScopeOpts(ctx)` maps the same precedence ladder onto `runThink`'s public options (`allowedSources`/`sourceId`) so the `think` op's gather and trajectory stages inherit the caller's source grant. `put_page`'s inline disk write-through is the shared `writePageThrough` helper (`src/core/write-through.ts`), ATOMIC via temp-sibling + rename so a crash or concurrent `gbrain sync` can't read a half-written `.md`; same helper backs `gbrain brainstorm/lsd --save`. Link provenance surface (#1941): `add_link` (`gbrain link`/`link-add`) + `remove_link` (`gbrain unlink`/`link-rm`) expose `link_source`/`link_type`; `add_link` rejects the reconciliation-managed built-ins via `MANAGED_LINK_SOURCES` (`markdown`/`frontmatter`/`mentions`/`wikilink-resolved`) and defaults omitted provenance to `'manual'` (the engine's own default stays `'markdown'` for internal callers); `list_link_sources` (`gbrain link-sources`, read) lists provenances via `sourceScopeOpts`. CLI aliases register through `cliHints.aliases` (collision-guarded in `src/cli.ts`). +- `src/core/operations.ts` — Contract-first operation definitions (the foundation). Exports upload validators `validateUploadPath`, `validatePageSlug`, `validateFilename`, plus `matchesSlugAllowList(slug, prefixes)` (glob matcher: `/*` matches recursive children; bare `` matches exact only). `OperationContext.remote` is a REQUIRED field flagging untrusted callers; `OperationContext.allowedSlugPrefixes` is the trusted-workspace allow-list set by the dream cycle; `OperationContext.auth?: AuthInfo` is threaded through HTTP dispatch for scope enforcement in `serve-http.ts` before the op runs. `enforceSubagentSlugFence(ctx, slug, opName)` is the shared fail-closed subagent write fence: when `viaSubagent` and `allowedSlugPrefixes` is set, the slug must match the allow-list; else the legacy `wiki/agents//...` namespace check applies. Both `put_page` and `add_timeline_entry` (subagent-allowlisted) route through it. Auto-link skipped only when `remote=true && !trustedWorkspace`. Every `Operation` carries `scope?: 'read' | 'write' | 'admin'` + `localOnly?: boolean`; `sync_brain`, `file_upload`, `file_list`, `file_url` are `admin + localOnly` (rejected over HTTP). Four trust-boundary call sites (`put_page` allowlist, file_upload trust-narrowing, submit_job protected-name guard, auto-link skip) use FAIL-CLOSED semantics: `ctx.remote === false` for trusted-only sites, `ctx.remote !== false` for "untrust unless explicit-false" — anything not strictly `false` is treated as remote (closes the HTTP MCP shell-job RCE where a read+write OAuth token could submit `shell` jobs). `sourceScopeOpts(ctx)` encodes the source-scoped read precedence ladder — federated array (`ctx.auth.allowedSources`) wins over scalar (`ctx.sourceId`/`ctx.auth.sourceId`) over nothing; every read-side op handler routes through it so a source-bound OAuth client can't see neighboring sources via `search`/`query`/`list_pages`/`get_page`/`find_experts`/query's image path, plus the by-slug reads `get_tags`/`get_links`/`get_backlinks`/`get_timeline` (and `get_page`'s tag fetch, which resolves against the concrete page's own `source_id`). `linkReadScopeOpts(ctx)` is the link-read sibling for `get_links`/`get_backlinks`: a link row references three pages (from/to/origin), and the engine's federated (`sourceIds[]`) branch scopes ALL THREE while its scalar (`sourceId`) branch scopes only the near endpoint (by design — trusted internal callers like `reconcileLinks` and back-link validators call the engine with a scalar scope and need the cross-source view). For an UNTRUSTED remote caller (`ctx.remote !== false`) carrying only a scalar scope it promotes that scope to a single-element `sourceIds:[id]`, routing them through the all-endpoint branch so a far/origin slug from outside the grant isn't disclosed; a federated array passes through unchanged and trusted local CLI (`ctx.remote === false`) keeps the scalar cross-source view. `thinkSourceScopeOpts(ctx)` maps the same precedence ladder onto `runThink`'s public options (`allowedSources`/`sourceId`) so the `think` op's gather and trajectory stages inherit the caller's source grant. `put_page`'s inline disk write-through is the shared `writePageThrough` helper (`src/core/write-through.ts`), ATOMIC via temp-sibling + rename so a crash or concurrent `gbrain sync` can't read a half-written `.md`; same helper backs `gbrain brainstorm/lsd --save`. Link provenance surface (#1941): `add_link` (`gbrain link`/`link-add`) + `remove_link` (`gbrain unlink`/`link-rm`) expose `link_source`/`link_type`; `add_link` rejects the reconciliation-managed built-ins via `MANAGED_LINK_SOURCES` (`markdown`/`frontmatter`/`mentions`/`wikilink-resolved`) and defaults omitted provenance to `'manual'` (the engine's own default stays `'markdown'` for internal callers); `list_link_sources` (`gbrain link-sources`, read) lists provenances via `sourceScopeOpts`. CLI aliases register through `cliHints.aliases` (collision-guarded in `src/cli.ts`). - `src/core/engine.ts` — Pluggable engine interface (BrainEngine). `clampSearchLimit(limit, default, cap)` takes an explicit cap so per-operation caps can be tighter than `MAX_SEARCH_LIMIT`. Exports `LinkBatchInput`/`TimelineBatchInput` for the bulk-insert API (`addLinksBatch`/`addTimelineEntriesBatch`). `readonly kind: 'postgres' | 'pglite'` discriminator lets `src/core/migrate.ts` and others branch without `instanceof` + dynamic imports. Methods: `batchLoadEmotionalInputs(slugs?)` (CTE-shaped read with per-table aggregates so page × N tags × M takes never produces N×M rows), `setEmotionalWeightBatch(rows)` (`UPDATE FROM unnest($1::text[],$2::text[],$3::real[])` composite-keyed on `(slug, source_id)`), `getRecentSalience(opts)`, `findAnomalies(opts)`. `PageFilters` has `sort?: 'updated_desc'|'updated_asc'|'created_desc'|'slug'` + `PAGE_SORT_SQL` whitelist consumed by both engines. `listAllPageRefs(): Promise>` ordered by `(source_id, slug)` — cheap cross-source enumeration replacing the `getAllSlugs()→getPage(slug)` N+1 (which silently defaulted to `source_id='default'`); parity across postgres-engine.ts + pglite-engine.ts; Pinned by `test/e2e/multi-source-bug-class.test.ts`. `SearchOpts`+`PageFilters` add `sourceIds?: string[]` (federated read axis; both engines apply `WHERE source_id = ANY($N::text[])` when set, preserve scalar `sourceId` fast path when unset); `traverseGraph(slug, depth, opts?)` and `traversePaths(slug, opts?)` accept `opts.sourceId`/`opts.sourceIds`. The by-slug read methods carry the same federated axis: `getTags`/`getLinks`/`getBacklinks` opts and `TimelineOpts` (consumed by `getTimeline`) accept `sourceIds?: string[]` taking precedence over the scalar `sourceId` (`source_id = ANY($::text[])` scoping the slug→page-id lookup); the link reads (`getLinks`/`getBacklinks`) scope ALL THREE endpoints (from/to/origin) on the federated branch while the scalar branch scopes only the near endpoint for trusted internal cross-source callers. `traverseGraph` opts has `frontierCap?: number` (per-iteration recursive-CTE cap, approx per-BFS-layer); return type `Promise` for MCP wire stability; export `TraverseGraphOpts`; Postgres uses parenthesized `LIMIT N ORDER BY (slug, id)` inside the recursive term, PGLite mirrors with positional params; Pinned by `test/regressions/v0_36_frontier_cap.test.ts`. Phantom-redirect methods: `refreshPageBody(slug, sourceId, compiled_truth, timeline, content_hash)` narrow-UPDATEs three columns + updated_at, skipping soft-deleted rows (content_hash refresh required so `gbrain sync` sees the canonical as unchanged after fence merge); `migrateFactsToCanonical(phantomSlug, canonicalSlug, sourceId)` UPDATEs `entity_slug`+`source_markdown_slug` on every active fact row keyed on the phantom, preserving embedding/validUntil/kind/status/source_session/confidence; parity at `test/phantom-redirect-engine-parity.test.ts`. `getAdjacencyBoosts(pageIds): Promise>` powers the per-query graph-signals stage — one SQL query returning inbound-link counts among top-K plus a cross-source count (links from differing `source_id`); `COALESCE(p.source_id,'default')` null safety, `HAVING >= 1`, cross-source CASE-WHEN excludes the target's own source; parity SQL across both engines; `SearchResult` gains optional `base_score`, `backlink_boost`, `salience_boost`, `recency_boost`, `exact_match_boost`, `graph_adjacency_boost`, `graph_cross_source_boost`, `session_demote_factor`, `reranker_delta` + internal staging fields; Pinned by `test/e2e/graph-signals-engine.test.ts`. Two REQUIRED methods: `deletePages(slugs, {sourceId}): Promise` (single-batch primitive returning slugs actually deleted) and `resolveSlugsByPaths(paths, {sourceId}): Promise>` (batch path→slug lookup); `sourceId` REQUIRED on both at the type level (asymmetric with single-row `deletePage` which keeps optional/'default'); both short-circuit on empty input and throw when `> DELETE_BATCH_SIZE`. Embedding-signature stale-detection quartet: `countStaleChunks(opts?)` gains optional `signature?: string` widening the stale predicate from `embedding IS NULL` to ALSO include chunks whose JOINed page `embedding_signature IS NOT NULL AND <> $signature` (NULL signature is GRANDFATHERED, never counted; omit `signature` for the legacy NULL-only count); `sumStaleChunkChars(opts?: {sourceId?, signature?}): Promise` = `SUM(LENGTH(chunk_text))` over stale chunks (same predicate + embed_skip filter + optional sourceId scope), used by `gbrain sync --all` cost preview via `estimateCostFromChars`; `setPageEmbeddingSignature(slug, {sourceId?, signature})` stamps `pages.embedding_signature` after a page's chunks (re)embed, idempotent no-op when page absent; `invalidateStaleSignatureEmbeddings({signature, sourceId?}): Promise` NULLs `embedding`+`embedded_at` on every chunk whose page signature is set AND differs, returning the count, called BEFORE `listStaleChunks` so signature-drift pages flow through the NULL-embedding keyset cursor unchanged (NULL never invalidated). Widens `findOrphanPages(opts?: {sourceId?, sourceIds?})` (candidate-side scoping only; inbound links counted from any source). Pinned by `test/sum-stale-chunk-chars.test.ts`, `test/embedding-signature-stale.test.ts`, `test/e2e/engine-parity.test.ts`. Free-text alias layer: `resolveAliases(aliasNorms, opts?): Promise>>` (READ; maps each normalized alias to declaring `(slug, source_id)` pairs, source-scoped) and `setPageAliases(slug, sourceId, aliasNorms)` (WRITE; replaces the full alias set, delete-then-insert, empty clears, idempotent on the unique triple), called by the `importFromContent` ingest projection and the `reindex --aliases` backfill; parity across both engines, Pinned by `test/search/page-aliases-engine.test.ts`. `searchVector` in both engines injects the shared `buildBestPerPagePoolCte` per-page max-pool so a page surfaces on its strongest chunk. `executeRawDirect(sql, params?, opts?)` is the lock-hot-path sibling of `executeRaw`: same single-statement contract, but routes to the direct session-mode pool when dual-pool is active (Postgres/Supabase port 5432) so a long-held lock heartbeat survives the transaction pooler's per-transaction connection recycling; PGLite delegates straight to `executeRaw` (no pooler). Both engines implement it; the Minion lock path (`claim`/`renewLock`) is the consumer. `reconnect(ctx?: {error?})` is a REQUIRED lifecycle method on BOTH engines: it recovers a dropped connection using the config captured at the last `connect()`, so callers (autopilot health probe, `batchRetry`) never `disconnect()` + bare `connect()` (which loses the config and throws `database_url undefined` forever, and opens a null-connection window). PostgresEngine rebuilds its pool with a `_reconnecting` reentrancy guard and emits pool-recovery audit; PGLite is single-writer in-process so it just restores the saved data dir for parity. - `src/core/engine-constants.ts` — single source of truth for engine batch-sizing constants. Exports `DELETE_BATCH_SIZE = 500` consumed by both engines' `deletePages` + `resolveSlugsByPaths` and by the sync delete + rename loops. Lives outside `engine.ts` (the interface module) to avoid circular-import worry — bounded per-statement work for predictable lock hold time + write amplification. - `src/core/background-work.ts` (#1762/#1745/#1775) — process background-work registry: the single owner of "drain every fire-and-forget DB-write sink before the CLI disconnects." `registerBackgroundWorkDrainer({name, order, drain(timeoutMs), abort?})` + `drainAllBackgroundWorkForCliExit({timeoutMs})` over a `Map` (idempotent registration by name; `__registerDrainerForTest` returns an unregister handle). Drains in explicit `(order, name)` order — facts FIRST (order 0) so its abort-path DB `logIngest` runs against the freshest live engine — and AWAITS `abort()` only when `drain()` reports `unfinished>0`. Best-effort per drainer: one sink's failure never blocks the others or the disconnect. FIVE sinks register at module import: `facts/queue.ts` (order 0; `abort`=`shutdown()` cancels a hung facts:absorb Haiku via internalAbort), `last-retrieved.ts` (order 1), `search/hybrid.ts` (order 2; `awaitPendingSearchCacheWrites` bounded via `Promise.race`), `eval-capture.ts` (order 3; `captureEvalCandidate` self-tracks its promise via `awaitPendingEvalCaptures`), `context/volunteer-events.ts` (order 4, #2095 — batched volunteer-event INSERTs, drained like the rest). Every cli.ts teardown site reaches it through `finishCliTeardown` (`src/core/cli-force-exit.ts`), which drains the registry before `engine.disconnect()` — closing the PGLite busy-loop where `db.close()` raced an in-flight job and pinned the single-writer lock (#1762). Exports `backgroundWorkSinkCount()` so the teardown helper computes its backstop deadline from the registered sink count. CLI-EXIT-ONLY: the facts `shutdown()` abort is permanent process state, never call in a long-lived `gbrain serve`. Companion changes: `src/core/ai/gateway.ts` `withDefaultTimeout(caller, ms)` bounds every outbound AI call (chat 300s, embed+multimodal 60s; env `GBRAIN_AI_{CHAT,EMBED,MULTIMODAL}_TIMEOUT_MS`; composed with caller signals via `AbortSignal.any`) and the teardown backstop honors an errored op's exit code so a hung disconnect can't mask failure as success (see `cli-force-exit.ts`); `src/core/postgres-engine.ts` `reconnect()` module-mode branch re-establishes via idempotent `db.connect()` + `connectionManager.setReadPool` refresh instead of `db.disconnect()` (no null window for concurrent ops; fail-loud on real connect failure — #1745); `src/core/search/hybrid.ts` `embedQueryBounded` + a shared `QueryEmbedDeadline` (6s, floored 2s per embed via `MIN_QUERY_EMBED_BUDGET_MS`; env `GBRAIN_QUERY_EMBED_TIMEOUT_MS`) bounds the cache-lookup AND inner query embeds so a stalled provider falls back to keyword instead of stalling the whole op (#1775). Incorporates + hardens PR #1763 (@ElliotDrel). Pinned by `test/core/background-work.test.ts`, `test/search/query-embed-deadline.test.ts`, `test/eval-capture-drain.test.ts`, `test/e2e/postgres-reconnect-singleton.test.ts`, `test/e2e/pglite-cli-exit.serial.test.ts`, `test/fix-wave-structural.test.ts`. @@ -90,7 +90,7 @@ per-release `**vX.Y.Z:**` narration — CI enforces this - `src/commands/autopilot.ts` extension — tick body invokes `runNightlyQualityProbe` when `cfg.autopilot.nightly_quality_probe.enabled === true` (default OFF — opt-in to protect API spend). NO scheduler-side rate-limit check — `runNightlyQualityProbe`'s internal `shouldRunNightly` (reading the audit JSONL) is the single source of truth. Probe call wrapped in try/catch that logs via `logError` and does NOT bump `consecutiveErrors` (probe failure is informational, never crashes the loop). Default `max_usd` cap = 5. Pinned by `test/autopilot-nightly-probe-wiring.test.ts`. - `test/eval-replay-gate.test.ts` + `test/fixtures/eval-baselines/qrels-search.json` — hermetic retrieval qrels gate running in the standard PR unit-shard CI matrix (`.github/workflows/test.yml`, NOT the fixed-file E2E workflow). Uses the canonical PGLite block (test-isolation R3+R4) and the basis-vector embedding pattern from `test/e2e/search-quality.test.ts:23-28` for fully hermetic retrieval. The qrels fixture (12 queries) uses PLACEHOLDER names only (alice-example, widget-co-example, etc. — privacy rule) and embeds each query at a deterministic basis dimension so retrieval is reproducible. Each query lists `relevant_slugs[]` + `first_relevant_slug`; the test computes `top1_match_rate` (top-1 == first_relevant) and `recall@10` (fraction of relevant_slugs in top-10), asserting both meet floors (defaults `>= 0.80` and `>= 0.85`). Env-overridable floors `GBRAIN_REPLAY_GATE_TOP1_FLOOR` / `GBRAIN_REPLAY_GATE_RECALL_FLOOR` (via `withEnv()` per R1). Refresh discipline: when ranking changes intentionally move expected slugs, edit `qrels-search.json` directly with a `Why:` line in the commit body or the gate degrades to rubber-stamp. Pinned by `test/eval-replay-gate.test.ts` (incl. a privacy-grep regression guard against real-name reintroduction). - `src/core/cycle/nightly-quality-probe.ts` + `src/core/audit-quality-probe.ts` + `test/fixtures/longmemeval-nightly.jsonl` + `test/nightly-quality-probe.test.ts` — opt-in nightly cross-modal quality probe. The phase runs `gbrain eval longmemeval --by-type` against the committed 10-question placeholder fixture, pipes output through `gbrain eval cross-modal --batch --max-usd 5 --yes`, and writes one event per run to `~/.gbrain/audit/quality-probe-YYYY-Www.jsonl` (ISO-week-rotated, mirrors `audit-slug-fallback.ts`; honors `GBRAIN_AUDIT_DIR`). Default DISABLED — opt-in via `gbrain config set autopilot.nightly_quality_probe.enabled true` (prevents surprise API spend). 24h rate limit (pure `shouldRunNightly(now, recentEvents, windowMs?)`) skips with audit row `outcome: rate_limited`. Embedding-key short-circuit: longmemeval needs `gateway.embedQuery()`, so the phase exits early with `outcome: no_embedding_key` + stderr warn when no provider configured. Full DI surface via `NightlyProbeDeps` (`isEnabled`, `hasEmbeddingProvider`, `resolveMaxUsd`, `resolveRepoRoot`, `runLongMemEval`, `runCrossModalBatch`, `now`) so the unit test stubs every external effect. Cost ceiling: $5/run × 30 nights ≈ $150/month worst-case; expected ~$10.50/month. New `nightly_quality_probe_health` doctor check (`src/commands/doctor.ts`, right after `slug_fallback_audit`) reads last 7 days: SKIPPED when flag off (with enable command); OK when enabled + all PASS; WARN on any FAIL / ERROR / BUDGET_EXCEEDED with per-outcome counts. Pinned by `test/nightly-quality-probe.test.ts`. -- `src/commands/eval-trajectory.ts` + `src/commands/founder-scorecard.ts` + `src/core/trajectory.ts` — temporal trajectory + founder scorecard. `gbrain eval trajectory ` shows the chronological typed-claim history (mrr/arr/team_size/etc) with regressions auto-flagged inline; `gbrain founder scorecard ` rolls up claim_accuracy / consistency / growth_trajectory / red_flags into one JSON. Pure-function math in `trajectory.ts`: `detectRegressions(points, threshold)` walks consecutive metric-value pairs per metric (10% drop default, env override `GBRAIN_TRAJECTORY_REGRESSION_THRESHOLD`); `computeDriftScore(points)` returns `1 - mean(cosine(emb[i], emb[i-1]))` over existing embeddings (null when <3 embedded points). Backed by `BrainEngine.findTrajectory(opts)` — both Postgres and PGLite, single SQL query, deterministic `ORDER BY valid_from ASC, id ASC`. Source-scoped via the `sourceId` scalar / `sourceIds` array dual pattern; visibility-filtered for remote callers. MCP op `find_trajectory` (read scope, NOT localOnly) registered after `find_experts`. Migration v67 adds optional typed-claim columns (`claim_metric`, `claim_value`, `claim_unit`, `claim_period`) + a partial index on `(entity_slug, claim_metric, valid_from) WHERE claim_metric IS NOT NULL`. Fence widens from 10 to 14 cells when any row has typed data; renderer stays at 10 cells when none do (no churn diff on existing fences). Metric labels normalize to lowercase snake_case via `normalizeMetricLabel` (15-entry seed map). The `consolidate` cycle phase does semantic upsert keyed on `(page_id, claim, since_date)` (fixes the duplicate-takes bug where re-running the cycle after `extract_facts` cleared `consolidated_at` appended duplicates via `MAX(row_num)+1`) and writes chronological `valid_until` on each cluster's older facts. The `extract_facts` cycle phase batch-embeds via `gateway.embed()` before insert AND threads `pages.effective_date` as the `pageEffectiveDate` fallback for `valid_from` (precedence: fence-row > pageEffectiveDate > now()). The contradiction probe MUST NOT write `valid_until` — grep guard at `test/eval-contradictions/no-valid-until-write.test.ts`. Haiku extraction lives in `src/core/facts/extract.ts` (not the `extract-facts.ts` cycle phase); `pageEffectiveDate` is OPTIONAL because `fence-write.ts` callers have no Page object. Migration v89 adds a nullable `event_type TEXT` column on `facts` so the substrate carries event-shaped rows (`event_type='meeting'` / `'job_change'` / `'location_change'`) alongside metric rows. `TrajectoryPoint.event_type: string | null` projected by both engines. `TrajectoryOpts.kind?: 'metric' | 'event' | 'all'` filter (default `'all'`); `founder-scorecard` + `eval-trajectory` pass `kind: 'metric'` explicitly. Back-compat pinned by `test/regressions/v0_40_2_0-trajectory-backcompat.test.ts` (byte-identical `computeFounderScorecard` + `computeTrajectoryStats` with and without event rows); engine parity in `test/engine-parity-event-type.test.ts`. +- `src/commands/eval-trajectory.ts` + `src/commands/founder-scorecard.ts` + `src/core/trajectory.ts` — temporal trajectory + founder scorecard. `gbrain eval trajectory ` shows the chronological typed-claim history (mrr/arr/team_size/etc) with regressions auto-flagged inline; `gbrain founder scorecard ` rolls up claim_accuracy / consistency / growth_trajectory / red_flags into one JSON. Pure-function math in `trajectory.ts`: `detectRegressions(points, threshold)` walks consecutive metric-value pairs per metric (10% drop default, env override `GBRAIN_TRAJECTORY_REGRESSION_THRESHOLD`); `computeDriftScore(points)` returns `1 - mean(cosine(emb[i], emb[i-1]))` over existing embeddings (null when <3 embedded points). Backed by `BrainEngine.findTrajectory(opts)` — both Postgres and PGLite, single SQL query, deterministic `ORDER BY valid_from ASC, id ASC`. Source-scoped via the `sourceId` scalar / `sourceIds` array dual pattern; visibility-filtered for remote callers. MCP op `find_trajectory` (read scope, NOT localOnly) registered after `find_experts`. Migration v67 adds optional typed-claim columns (`claim_metric`, `claim_value`, `claim_unit`, `claim_period`) + a partial index on `(entity_slug, claim_metric, valid_from) WHERE claim_metric IS NOT NULL`. Fence widens from 10 to 14 cells when any row has typed data; renderer stays at 10 cells when none do (no churn diff on existing fences). Metric labels normalize to lowercase snake_case via `normalizeMetricLabel` (15-entry seed map). The `consolidate` cycle phase does semantic upsert keyed on `(page_id, claim, since_date)` (fixes the duplicate-takes bug where re-running the cycle after `extract_facts` cleared `consolidated_at` appended duplicates via `MAX(row_num)+1`) and writes chronological `valid_until` on each cluster's older facts. The `extract_facts` cycle phase batch-embeds via `gateway.embed()` before insert AND threads `pages.effective_date` as the `pageEffectiveDate` fallback for `valid_from` (precedence: fence-row > pageEffectiveDate > now()). The contradiction probe MUST NOT write `valid_until` — grep guard at `test/eval-contradictions/no-valid-until-write.test.ts`. Haiku extraction lives in `src/core/facts/extract.ts` (not the `extract-facts.ts` cycle phase); its output cap is config `facts.extraction_max_tokens` (default 4000), a `stopReason: 'length'` response retries once at 2× the cap, and persistent truncation warns loudly on stderr instead of silently extracting zero facts; `pageEffectiveDate` is OPTIONAL because `fence-write.ts` callers have no Page object. Migration v89 adds a nullable `event_type TEXT` column on `facts` so the substrate carries event-shaped rows (`event_type='meeting'` / `'job_change'` / `'location_change'`) alongside metric rows. `TrajectoryPoint.event_type: string | null` projected by both engines. `TrajectoryOpts.kind?: 'metric' | 'event' | 'all'` filter (default `'all'`); `founder-scorecard` + `eval-trajectory` pass `kind: 'metric'` explicitly. Back-compat pinned by `test/regressions/v0_40_2_0-trajectory-backcompat.test.ts` (byte-identical `computeFounderScorecard` + `computeTrajectoryStats` with and without event rows); engine parity in `test/engine-parity-event-type.test.ts`. - `src/core/trajectory-format.ts` — shared `formatTrajectoryBlock(points, entitySlug, opts)` consumed by both `gbrain think` (production) and the LongMemEval harness (benchmark). Groups by `(metric ?? event_type)`, per-metric cap 20, total cap 100, knowledge_update intent annotates value-change rows with `(superseded prior)`. Emits a `` XML envelope — `INJECTION_PATTERNS` in `src/core/think/sanitize.ts` escapes ``, `` open tags, and attribute injection so adversarial fact text can't break out. Pinned by `test/trajectory-format.test.ts`. - `src/core/think/intent.ts` + `src/core/think/entity-extract.ts` — pure `classifyIntent(question)` returns `'temporal' | 'knowledge_update' | 'other'` (regex-first, no LLM, `'other'` fast path short-circuits with zero SQL). `extractCandidateEntities(question, retrievedSlugs)` pulls high-precision candidates from retrieved entity-prefix slugs (`people/`, `companies/`, `organizations/`) and medium-precision noun phrases. Stop-word boundaries + leading-verb stripper handle "When did I last meet Marco" → `marco`. Both consumed by `runThink` and the LongMemEval harness so the two paths cannot drift. Pinned by `test/think-intent.test.ts` and `test/think-entity-extract.test.ts`. - `src/commands/eval-suspected-contradictions.ts` + `src/core/eval-contradictions/{judge,runner,types,date-filter,cost-tracker,cache,severity-classify,cross-source,trends,calibration,judge-errors,auto-supersession,fixture-redact}.ts` — `gbrain eval suspected-contradictions [run|trend|review]`. Probe samples top-K retrieval pairs per query (cross-slug + intra-page chunk-vs-take), date pre-filters (3-rule layered — same-paragraph-dual-date overrides separation rule), LLM judge (query-conditioned; UTF-8-safe truncation; confidence-floor double-enforcement; resolution_kind output drives paste-ready commands), persistent cache keyed on `(chunk_a_hash, chunk_b_hash, model_id, prompt_version, truncation_policy)` (prompt edits cleanly invalidate prior verdicts), Wilson 95% CI calibration on the headline percentage with `small_sample_note` when n<30, judge_errors as first-class typed counters (parse_fail/refusal/timeout/http_5xx/unknown — avoids bias from silent skip), trend writes to `eval_contradictions_runs`, source-tier breakdown reuses `DEFAULT_SOURCE_BOOSTS` prefix logic, deterministic sampling (combined_score DESC + lex tiebreaker for stable cache hit-rate). Hermetic via `judgeFn` + `searchFn` DI in the runner; never touches the real gateway in tests. Engine surface: `BrainEngine.listActiveTakesForPages` (batched), `writeContradictionsRun` + `loadContradictionsTrend`, `getContradictionCacheEntry` + `putContradictionCacheEntry` + `sweepContradictionCache`. Schema migrations v51 + v52. MCP op `find_contradictions` (read scope, NOT localOnly, NOT in subagent allowlist — user-initiated only). Doctor check surfaces high-severity findings with paste-ready resolution commands; synthesize phase pre-fetches the latest probe's top-5-by-severity findings and threads them into `buildSynthesisPrompt` as an informational block. Architecture doc: `docs/contradictions.md`. @@ -247,14 +247,14 @@ per-release `**vX.Y.Z:**` narration — CI enforces this - `src/core/minions/handlers/shell-audit.ts` — per-submission JSONL audit trail at `~/.gbrain/audit/shell-jobs-YYYY-Www.jsonl` (ISO-week rotation; override via `GBRAIN_AUDIT_DIR`). Best-effort: `mkdirSync(recursive)` + `appendFileSync`; failures logged to stderr, submission not blocked. Logs cmd (first 80 chars) or argv (JSON array). Never logs env values. - `src/core/minions/handlers/supervisor-audit.ts` — supervisor lifecycle JSONL audit at `~/.gbrain/audit/supervisor-YYYY-Www.jsonl` (ISO-week rotation; shares `computeIsoWeekName()` with `shell-audit.ts`). `writeSupervisorEvent(emission, supervisorPid)` appends one line per event (`started`, `worker_spawned`, `worker_exited`, `backoff`, `health_warn`, `health_error`, `max_crashes_exceeded`, `shutting_down`, `stopped`, `worker_spawn_failed`). `readSupervisorEvents({sinceMs})` is the readback for `gbrain doctor`. Exports `isCrashExit(event)`, `summarizeCrashes(events)`, `CrashSummary` type, and `CLEAN_EXIT_CAUSES` denylist (`'clean_exit' | 'graceful_shutdown'`). Single regression point — both `gbrain doctor` (supervisor check at `doctor.ts:1011-1043`) and `gbrain jobs supervisor status` (`jobs.ts:803-826`) import from here so the two surfaces can't drift. `isCrashExit` classifies a single `worker_exited` against the denylist: clean/graceful are NON-crashes; everything else (incl. any future `likely_cause` from `child-worker-supervisor.ts`) is a crash; audit lines lacking `likely_cause` fall back to `code !== 0`. `summarizeCrashes` returns `{total, by_cause: {runtime_error, oom_or_external_kill, unknown, legacy}, clean_exits}` — the `legacy` bucket catches both old fallback entries AND unrecognized future causes (fail-loud, not silent underreport); denylist-over-allowlist is deliberate. Pinned by `test/supervisor-audit.test.ts` (14 cases) and 4 source-grep wiring assertions in `test/doctor.test.ts`. - `src/core/minions/backpressure-audit.ts` — sibling of shell-audit.ts for `maxWaiting` coalesce events. JSONL at `~/.gbrain/audit/backpressure-YYYY-Www.jsonl`. One line per coalesce with `(queue, name, waiting_count, max_waiting, returned_job_id, ts)`. Closes the silent-drop vector the maxWaiting guard introduced. -- `src/core/minions/handlers/subagent.ts` — LLM-loop handler. Two-phase tool persistence (pending → complete/failed), replay reconciliation for mid-dispatch crashes, dual-signal abort (`ctx.signal` + `ctx.shutdownSignal`), Anthropic prompt caching on system + tool defs. `makeSubagentHandler({engine, client?, ...})` factory; `MessagesClient` is an injectable interface the real SDK implements structurally. Throws `RateLeaseUnavailableError` (renewable) when rate-lease capacity is full. Anthropic 400 `prompt is too long` responses (status 400 + body matches `/prompt is too long|prompt_too_long|context.*length/i`) classify as `UnrecoverableError` so the job goes straight to `dead` on first attempt instead of stalling three times. Catches both initial-prompt overflow and turn-N tool-loop accumulation that `synthesize.ts`'s chunker can't bound ahead of time. +- `src/core/minions/handlers/subagent.ts` — LLM-loop handler. Two-phase tool persistence (pending → complete/failed), replay reconciliation for mid-dispatch crashes, dual-signal abort (`ctx.signal` + `ctx.shutdownSignal`), Anthropic prompt caching on system + tool defs. `makeSubagentHandler({engine, client?, ...})` factory; `MessagesClient` is an injectable interface the real SDK implements structurally. Per-turn output cap resolves via `resolveMaxOutputTokens` (`data.max_tokens` → `agent.max_output_tokens` config → 8192 default); a `stop_reason: 'max_tokens'` final turn surfaces as `SubagentStopReason 'max_tokens'` (not a silent `end_turn`), and a max_tokens stop mid-tool-round injects a truncation note into the tool-result turn so the model re-issues the dropped call. Throws `RateLeaseUnavailableError` (renewable) when rate-lease capacity is full. Anthropic 400 `prompt is too long` responses (status 400 + body matches `/prompt is too long|prompt_too_long|context.*length/i`) classify as `UnrecoverableError` so the job goes straight to `dead` on first attempt instead of stalling three times. Catches both initial-prompt overflow and turn-N tool-loop accumulation that `synthesize.ts`'s chunker can't bound ahead of time. - `src/core/minions/handlers/subagent-aggregator.ts` — `subagent_aggregator` handler. Claims AFTER all children resolve (queue guarantees every terminal child posts a `child_done` inbox message with outcome). Reads inbox via `ctx.readInbox()`, builds a deterministic mixed-outcome markdown summary. No LLM call. - `src/core/minions/handlers/subagent-audit.ts` — JSONL audit + heartbeat writer at `~/.gbrain/audit/subagent-jobs-YYYY-Www.jsonl`. Events: `submission` (one per submit) + `heartbeat` (per turn boundary: `llm_call_started | llm_call_completed | tool_called | tool_result | tool_failed`). Never logs prompts or tool inputs. `readSubagentAuditForJob(jobId, {sinceIso})` is the readback for `gbrain agent logs`. - `src/core/minions/rate-leases.ts` — lease-based concurrency cap for outbound providers (default key `anthropic:messages`, max via `GBRAIN_ANTHROPIC_MAX_INFLIGHT`). Owner-tagged rows with `expires_at` auto-prune on acquire; `pg_advisory_xact_lock` guards check-then-insert; CASCADE on owning job deletion. `renewLeaseWithBackoff` retries 3x (250/500/1000ms). - `src/core/minions/wait-for-completion.ts` — poll-until-terminal helper for CLI callers. `TimeoutError` does NOT cancel the job; `AbortSignal` exits without throwing. Default `pollMs`: 1000 on Postgres, 250 on PGLite inline. - `src/core/minions/transcript.ts` — renders `subagent_messages` + `subagent_tool_executions` to markdown. Tool rows splice under their owning assistant `tool_use` by `tool_use_id`. UTF-8-safe truncation; unknown block types fall through to fenced JSON. - `src/core/minions/plugin-loader.ts` — `GBRAIN_PLUGIN_PATH` discovery. Absolute paths only, left-wins collision, `gbrain.plugin.json` with `plugin_version: "gbrain-plugin-v1"`, plugins ship DEFS only (no new tools), `allowed_tools:` validated at load time against the derived registry. -- `src/core/minions/tools/brain-allowlist.ts` — derives the subagent tool registry from `src/core/operations.ts` (13-name allow-list). By default `put_page` schema is namespace-wrapped per subagent (`^wiki/agents//.+`). When `BuildBrainToolsOpts.allowedSlugPrefixes` is set, the put_page schema describes the prefix list to the model and the OperationContext is threaded with `allowedSlugPrefixes` — trust comes from `PROTECTED_JOB_NAMES` gating subagent submission (MCP cannot reach this field); only cycle.ts (synthesize/patterns) and direct CLI submitters set it. Allow-list includes `get_recent_salience` + `find_anomalies` but deliberately NOT `get_recent_transcripts` (all subagent calls run `ctx.remote === true` and the trust gate rejects remote callers, so it would always reject; the cycle synthesize phase calls `discoverTranscripts` directly instead). `paramsToInputSchema()` consumes `paramDefToSchema` from `src/mcp/tool-defs.ts`; required-aggregation at the tool-def level stays here (the shared helper is per-param). +- `src/core/minions/tools/brain-allowlist.ts` — derives the subagent tool registry from `src/core/operations.ts` (15-name allow-list, size pinned by `test/brain-allowlist.serial.test.ts`). Includes `add_timeline_entry` (the canonical timeline write), fenced server-side by the same `enforceSubagentSlugFence` policy as `put_page`. By default `put_page` schema is namespace-wrapped per subagent (`^wiki/agents//.+`). When `BuildBrainToolsOpts.allowedSlugPrefixes` is set, the put_page schema describes the prefix list to the model and the OperationContext is threaded with `allowedSlugPrefixes` — trust comes from `PROTECTED_JOB_NAMES` gating subagent submission (MCP cannot reach this field); only cycle.ts (synthesize/patterns) and direct CLI submitters set it. Allow-list includes `get_recent_salience` + `find_anomalies` but deliberately NOT `get_recent_transcripts` (all subagent calls run `ctx.remote === true` and the trust gate rejects remote callers, so it would always reject; the cycle synthesize phase calls `discoverTranscripts` directly instead). `paramsToInputSchema()` consumes `paramDefToSchema` from `src/mcp/tool-defs.ts`; required-aggregation at the tool-def level stays here (the shared helper is per-param). - `src/mcp/tool-defs.ts` — `buildToolDefs(ops)` helper; MCP server + subagent tool registry both call it, byte-for-byte equivalence pinned by `test/mcp-tool-defs.test.ts`. Exports the recursive `paramDefToSchema(p: ParamDef)` — single source of truth for ParamDef→JSON Schema mapping shared by three consumers: `buildToolDefs` (stdio MCP), `src/commands/serve-http.ts:837` (HTTP MCP `tools/list`), and `src/core/minions/tools/brain-allowlist.ts:84` (subagent registry). Recursive on `items` so nested array-of-arrays preserves inner shape on the wire. Key ordering (type, description, enum, default, items) is intentional so `JSON.stringify` output stays byte-stable. `test/mcp-tool-defs.test.ts` has a `findArrayWithoutItems` walker that fails on any `type: 'array'` lacking `items.type`. - `src/core/minions/attachments.ts` — Attachment validation (path traversal, null byte, oversize, base64, duplicate detection). - `src/commands/agent.ts` — `gbrain agent run [flags]` CLI. Submits `subagent` (or N children + 1 aggregator) under `{allowProtectedSubmit: true}`. Single-entry `--fanout-manifest` short-circuits. Children get `on_child_fail: 'continue'` + `max_stalled: 3`. `--follow` is the default on TTY; streams logs + polls `waitForCompletion` in parallel. Ctrl-C detaches, does not cancel. @@ -307,9 +307,9 @@ per-release `**vX.Y.Z:**` narration — CI enforces this - `src/core/import-checkpoint.ts` — `loadCheckpoint(brainDir)`, `saveCheckpoint(brainDir, completed)`, `resumeFilter(files, completed, brainDir)`, `clearCheckpoint()`, plus the `ImportCheckpoint` type. Path-set format `{schema_version, brainDir, completed: string[]}`. Atomic write via `.tmp` + `rename()` so a mid-write crash never leaves a partial JSON. `loadCheckpoint` returns `null` on: missing file, malformed JSON, brainDir mismatch (ran against a different brain), and the old positional format (logged to stderr before discard). `resumeFilter` returns `{toProcess, skippedCount}` — pure, no I/O, deterministic. `clearCheckpoint` is no-op-on-missing for clean-exit cleanup. Honors `GBRAIN_HOME` via `gbrainPath()` so `withEnv({GBRAIN_HOME: tmpdir})` test isolation works without monkey-patching fs. Best-effort persistence — `saveCheckpoint` logs warnings on write errors but never throws. - `src/core/sort-newest-first.ts` — single source of truth for the descending-lex sort that `gbrain import` and `gbrain sync` both apply. Mutates in place (Array.prototype.sort semantics), returns the same array reference for fluent chaining. Empty/single-element inputs short-circuit. Future ordering changes flip one line here instead of touching two CLI commands. Pinned by `test/sort-newest-first.test.ts` (descending order, mixed prefixes, empty, single-element, in-place-mutation contract). - `src/core/cycle.ts` — brain maintenance cycle primitive (9 phases). `runCycle(engine: BrainEngine | null, opts: CycleOpts): Promise` composes phases in semantic order: lint → backlinks → sync → synthesize → extract → patterns → recompute_emotional_weight → embed → orphans. `synthesize` runs after sync (cross-references see a fresh brain) and before extract (auto-link materializes its writes); `patterns` runs after extract so it reads a fresh graph (subagent put_page sets `ctx.remote=true` and skips auto-link/timeline by default, so extract is the canonical materialization); `recompute_emotional_weight` sees the union of `syncPagesAffected` + `synthesizeWrittenSlugs` incrementally, or all pages when neither anchor is set (full backfill via `gbrain dream --phase recompute_emotional_weight`). `CycleReport.schema_version: "1"` is stable; `totals` is additive (`pages_emotional_weight_recomputed`, `transcripts_processed`, `synth_pages_written`, `patterns_written`). Three callers: `gbrain dream` CLI, `gbrain autopilot` daemon inline path, the Minions `autopilot-cycle` handler. Coordination via `gbrain_cycle_locks` DB table + `~/.gbrain/cycle.lock` file lock with PID-liveness for PGLite. `yieldBetweenPhases` runs between phases; `yieldDuringPhase` is in-phase keepalive (synthesize/patterns renew the cycle-lock TTL during long waits). Engine nullable; lock-skip on read-only phase selections. `CycleOpts.signal?: AbortSignal` propagates the worker's abort signal with `checkAborted()` between every phase. `runPhaseSync` returns `pagesAffected` via `SyncPhaseResult` (threaded to `runPhaseExtract` as the 4th arg) and takes `willRunExtractPhase: boolean` setting `noExtract: phases.includes('extract')` so `gbrain dream --phase sync` doesn't silently lose extraction. `resolveSourceForDir(engine, brainDir)` threads `sourceId` to `performSync()` so sync reads the per-source `sources.last_commit` anchor (not the drift-prone global `config.sync.last_commit`). `CycleOpts.brainDir` is `string | null`; when null (checkout-less postgres/Supabase brain) the 6 filesystem phases (lint/backlinks/sync/synthesize/extract/patterns) skip with `details.reason: 'no_brain_dir'` and the DB-only phases run; `resolveSourceForDir` is null-tolerant. `cycleSourceId = opts.sourceId ?? resolveSourceForDir(engine, brainDir)` is the canonical per-source scope for `extract_facts`/`extract_atoms`/calibration so `gbrain dream --source repo-a` reconciles repo-a's facts even with no checkout (instead of scoping to `'default'` while stamping repo-a fresh). `deriveStatus` counts `edges_resolved`/`edges_ambiguous` as work so an edges-only cycle reports `ok` not `clean`; the `jobs.ts` `autopilot-cycle` + phase-wrapper handlers pass `null` (not `'.'`) when no repo is configured. The cycle is SPLIT for autopilot fan-out (#2194/#2227): `PHASE_SCOPE` partitions `ALL_PHASES` into `GLOBAL_PHASES` (brain-wide: embed/orphans/purge/resolve_symbol_edges/grade_takes/calibration_profile/synthesize_concepts/skillopt) and `NON_GLOBAL_PHASES` (source + mixed). Per-source `autopilot-cycle` jobs run only `NON_GLOBAL_PHASES` and stamp `last_source_cycle_at`; the single `autopilot-global-maintenance` job runs `GLOBAL_PHASES` (no `sourceId`) and stamps the brain-level `autopilot.last_global_at` config key (`LAST_GLOBAL_AT_KEY`). `last_full_cycle_at` is still written alongside `last_source_cycle_at` on a per-source success for doctor/legacy readers (no longer a gate for the brain-wide phases). Pinned by `test/dream-postgres.serial.test.ts` + `test/jobs-autopilot-cycle-braindir.serial.test.ts` + `test/autopilot-global-maintenance.test.ts`. -- `src/core/cycle/synthesize.ts` — Synthesize phase: conversation-transcript-to-brain pipeline. Reads `dream.synthesize.session_corpus_dir`, runs a cheap Haiku verdict (cached in `dream_verdicts`), then fans out one Sonnet subagent per worth-processing transcript with `allowed_slug_prefixes` (sourced from `skills/_brain-filing-rules.json` `dream_synthesize_paths.globs`). Orchestrator collects slugs from `subagent_tool_executions` (NOT `pages.updated_at`) and reverse-renders DB → markdown via `serializeMarkdown`. Cooldown via `dream.synthesize.last_completion_ts`, written ONLY on success. Idempotency key `dream:synth::`. `--dry-run` runs Haiku, skips Sonnet. Subagent never gets fs-write access. `renderPageToMarkdown` (exported) stamps `dream_generated: true` + `dream_cycle_date` into every reverse-write's frontmatter; `writeSummaryPage` does the same on the summary index — this marker is the explicit identity surface `isDreamOutput` checks in `transcript-discovery.ts`. `judgeSignificance` and `JudgeClient` are exported; `judgeSignificance` takes a `verdictModel` param loaded from `dream.synthesize.verdict_model` via `loadSynthConfig`. `splitTranscriptByBudget(content, contentHash, maxChars)` splits oversized transcripts at paragraph boundaries (`## Topic:` → `---` → `\n` ladder) using a deterministic offset seeded from the first 32 bits of `contentHash` so retries chunk identically; per-chunk char budget = `MODEL_CONTEXT_TOKENS[resolvedModel] × 0.9 × 3.5 chars/token` (non-Anthropic ids fall back to a 180K-token safe default + once-per-process stderr warn); operator overrides `dream.synthesize.max_prompt_tokens` (floor 100K, wins) and `dream.synthesize.max_chunks_per_transcript` (default 24). Per-chunk idempotency keys `dream:synth:::cof`; single-chunk transcripts preserve the legacy `dream:synth::` key byte-for-byte so existing brains skip with `already_synthesized_legacy_single_chunk` instead of re-spending Sonnet. `collectChildPutPageSlugs` raw-fetches every (job_id, slug) pair (not `SELECT DISTINCT`) and rewrites bare-hash6 slugs to `-c` for chunked children (orchestrator-side, zero Sonnet trust). Cap-hit skips don't write to `dream_verdicts` so raising the cap on next run re-attempts cleanly. Bounds INITIAL prompt size only; tool-loop turn-N accumulation is caught by terminal-error classification in `subagent.ts`. Verdict routing is gateway-routed: `makeJudgeClient(verdictModel)` (exported, replacing `makeHaikuClient()`) mirrors `tryBuildGatewayClient` in `src/core/think/index.ts` — a construction-time provider/key probe returns `null` on a clear miss (unknown provider id via `resolveRecipe` AIConfigError, or Anthropic provider with no key via `hasAnthropicKey()`). The verdict loop wraps `judgeSignificance` in try/catch for `AIConfigError` so mid-run provider failures surface as per-transcript `worth=false, reasons=['gateway error: ...']` instead of crashing the phase. Canonical config key `models.dream.synthesize_verdict` (per `PER_TASK_KEYS` in `src/core/model-config.ts`); `JudgeClient` signature preserved verbatim for test-seam stability; CI guard `scripts/check-gateway-routed-no-direct-anthropic.sh` prevents reintroducing `new Anthropic()` here or in `think/index.ts`. At the queue.add boundary (lines 395-404) a conditional `anthropic:` prefix is applied ONLY when the resolved model has no colon AND starts with `claude-` (because `resolveModel` returns bare ids from `TIER_DEFAULTS`/`DEFAULT_ALIASES` and the subagent validator requires `provider:model` form) — avoids changing the shared constants which would ripple across every `resolveModel` caller. Pinned by `test/cycle/synthesize-gateway-adapter.test.ts`, `test/e2e/dream-synthesize-pglite.test.ts` (gateway-adapter mid-run AIConfigError catch), `test/cycle/regression-pr-wave-r1-r2-r4.test.ts`. +- `src/core/cycle/synthesize.ts` — Synthesize phase: conversation-transcript-to-brain pipeline. Reads `dream.synthesize.session_corpus_dir`, runs a cheap Haiku verdict (cached in `dream_verdicts`), then fans out one Sonnet subagent per worth-processing transcript with `allowed_slug_prefixes` (sourced from `skills/_brain-filing-rules.json` `dream_synthesize_paths.globs`). Orchestrator collects slugs from `subagent_tool_executions` (NOT `pages.updated_at`) and reverse-renders DB → markdown via `serializeMarkdown`. Cooldown via `dream.synthesize.last_completion_ts`, written ONLY on success. Idempotency key `dream:synth::`. `--dry-run` runs Haiku, skips Sonnet. Subagent never gets fs-write access. `renderPageToMarkdown` (exported) stamps `dream_generated: true` + `dream_cycle_date` into every reverse-write's frontmatter; `writeSummaryPage` does the same on the summary index — this marker is the explicit identity surface `isDreamOutput` checks in `transcript-discovery.ts`. `judgeSignificance` and `JudgeClient` are exported; `judgeSignificance` takes a `verdictModel` param loaded from `dream.synthesize.verdict_model` via `loadSynthConfig`. `splitTranscriptByBudget(content, contentHash, maxChars)` splits oversized transcripts at paragraph boundaries (`## Topic:` → `---` → `\n` ladder) using a deterministic offset seeded from the first 32 bits of `contentHash` so retries chunk identically; per-chunk char budget = `MODEL_CONTEXT_TOKENS[resolvedModel] × 0.9 × 3.5 chars/token` (non-Anthropic ids fall back to a 180K-token safe default + once-per-process stderr warn); operator overrides `dream.synthesize.max_prompt_tokens` (floor 100K, wins) and `dream.synthesize.max_chunks_per_transcript` (default 24); per-chunk subagent job/wait timeouts are `dream.synthesize.subagent_timeout_ms` / `dream.synthesize.subagent_wait_timeout_ms` (defaults 30/35 min). Per-chunk idempotency keys `dream:synth:::cof`; single-chunk transcripts preserve the legacy `dream:synth::` key byte-for-byte so existing brains skip with `already_synthesized_legacy_single_chunk` instead of re-spending Sonnet. `collectChildPutPageSlugs` raw-fetches every (job_id, slug) pair (not `SELECT DISTINCT`) and rewrites bare-hash6 slugs to `-c` for chunked children (orchestrator-side, zero Sonnet trust). Cap-hit skips don't write to `dream_verdicts` so raising the cap on next run re-attempts cleanly. Bounds INITIAL prompt size only; tool-loop turn-N accumulation is caught by terminal-error classification in `subagent.ts`. Verdict routing is gateway-routed: `makeJudgeClient(verdictModel)` (exported, replacing `makeHaikuClient()`) mirrors `tryBuildGatewayClient` in `src/core/think/index.ts` — a construction-time provider/key probe returns `null` on a clear miss (unknown provider id via `resolveRecipe` AIConfigError, or Anthropic provider with no key via `hasAnthropicKey()`). The verdict loop wraps `judgeSignificance` in try/catch for `AIConfigError` so mid-run provider failures surface as per-transcript `worth=false, reasons=['gateway error: ...']` instead of crashing the phase. Canonical config key `models.dream.synthesize_verdict` (per `PER_TASK_KEYS` in `src/core/model-config.ts`); `JudgeClient` signature preserved verbatim for test-seam stability; CI guard `scripts/check-gateway-routed-no-direct-anthropic.sh` prevents reintroducing `new Anthropic()` here or in `think/index.ts`. At the queue.add boundary (lines 395-404) a conditional `anthropic:` prefix is applied ONLY when the resolved model has no colon AND starts with `claude-` (because `resolveModel` returns bare ids from `TIER_DEFAULTS`/`DEFAULT_ALIASES` and the subagent validator requires `provider:model` form) — avoids changing the shared constants which would ripple across every `resolveModel` caller. Pinned by `test/cycle/synthesize-gateway-adapter.test.ts`, `test/e2e/dream-synthesize-pglite.test.ts` (gateway-adapter mid-run AIConfigError catch), `test/cycle/regression-pr-wave-r1-r2-r4.test.ts`. - `scripts/check-gateway-routed-no-direct-anthropic.sh` — CI guard that fails the build if `src/core/cycle/synthesize.ts` or `src/core/think/index.ts` reintroduces a runtime `new Anthropic()` constructor call or a value-shaped `import Anthropic from '@anthropic-ai/sdk'` import. Type-only imports (`import type Anthropic from '@anthropic-ai/sdk'`) stay allowed for adapter types; comment lines (`//` or ` *` prefixes) are excluded so historical JSDoc doesn't false-fire. Mirrors `scripts/check-jsonb-pattern.sh`. Wired into `bun run verify` and `bun run check:all`. Extend `GUARDED_FILES` when migrating another file off direct SDK construction. -- `src/core/cycle/patterns.ts` — Patterns phase: cross-session theme detection over reflections within `dream.patterns.lookback_days` (default 30). Names a pattern only when ≥`dream.patterns.min_evidence` (default 3) reflections support it. Single Sonnet subagent; same allow-list path as synthesize. Runs AFTER `extract` so the graph is fresh. +- `src/core/cycle/patterns.ts` — Patterns phase: cross-session theme detection over reflections within `dream.patterns.lookback_days` (default 30). Names a pattern only when ≥`dream.patterns.min_evidence` (default 3) reflections support it. Single Sonnet subagent; same allow-list path as synthesize. Subagent job/wait timeouts are config keys `dream.patterns.subagent_timeout_ms` / `dream.patterns.subagent_wait_timeout_ms` (defaults 30/35 min, mirroring the `dream.synthesize.*` pair). The phase status reflects the child outcome: non-`complete` outcome with zero writes → `fail` (error code `PATTERNS_CHILD_`); non-`complete` with partial writes → `warn`. Runs AFTER `extract` so the graph is fresh. - `src/core/cycle/extract-facts.ts` — extract_facts cycle phase. Fence is canonical: per-page wipe (`deleteFactsForPage`) + reinsert from `parseFactsFence` + `extractFactsFromFenceText` + `engine.insertFacts`. #1928: the per-page wipe passes `excludeSourcePrefixes: ['cli:']` so conversation facts (written by `extract-conversation-facts`, on pages with NO `## Facts` fence to recreate them from) survive the reconcile instead of being deleted-with-nothing-to-reinsert. The destructive phase no longer inherits a failed sync's full-brain walk: `slugs: []` (a real incremental no-op) is distinguished from `slugs: undefined` (full-walk intent) by presence, not length. `runPhaseExtractFacts` (cycle.ts) surfaces a `warn` (`net_fact_deletion`) when the reconcile deletes at least `NET_DELETION_WARN_FLOOR` (50) more facts than it reinserts — the exact signature of the conversation-facts wipe, which previously read as a silent `ok`. Empty-fence guard refuses when legacy rows (`row_num IS NULL AND entity_slug IS NOT NULL`) pend backfill (status: warn, hint: `gbrain apply-migrations --yes`). A phantom-redirect pre-pass runs AFTER the legacy-row guard, BEFORE the main reconcile loop: when `opts.brainDir` is set, `runPhantomRedirectPass(engine, brainDir, sourceId, dryRun)` walks unprefixed-slug pages capped by `GBRAIN_PHANTOM_REDIRECT_LIMIT` (default 50). The pass returns `touched_canonicals` — canonical slugs whose disk fence merged with phantom rows; `runExtractFacts` UNIONs them into the main reconcile slug set so canonical's DB facts derive from the merged fence in the same cycle (handles phantom-had-only-on-disk-fence). `ExtractFactsResult` gains six phantom fields: `phantomsScanned`, `phantomsRedirected`, `phantomsAmbiguous`, `phantomsSkippedDrift`, `phantomsLockBusy`, `phantomsMorePending`. Three bubble to `CycleReport.totals` (`phantoms_redirected`, `phantoms_ambiguous`, `phantoms_skipped_drift`). - `src/core/fence-shared.ts` — shared pipe-table primitives for the `## Takes` (`takes-fence.ts`) and `## Facts` (`facts-fence.ts`) fences: `parseRowCells`, `isSeparatorRow`, `stripStrikethrough`, `parseStringCell`, `escapeFenceCell`. `parseRowCells` is escape-aware: `\|` stays inside its cell and decodes back to a literal `|` (exact inverse of `escapeFenceCell`), while any other backslash passes through verbatim so existing fence text (e.g. Windows paths) is byte-stable across a render → parse cycle. This matters because the fence is canonical and reconcile is wipe-and-reinsert — a pipe inside a cell shifting the column layout would corrupt every row behind it on the next reconcile. Pinned by the escape cases in `test/facts-fence.test.ts` + the full render → parse → reconcile round-trip in `test/e2e/facts-fence-reconcile-postgres.test.ts`. - `src/core/entities/resolve.ts` — Free-form entity name → canonical slug resolution. `resolveEntitySlug(engine, source_id, raw)`: exact slug → unambiguous bare-name prefix expansion across `people/-%` + `companies/-%` → high-specificity fuzzy match for multi-token input (pg_trgm @ 0.7 threshold) → deterministic `slugify` holding fallback. Bare-name collisions never use popularity as confidence; shared-token company names below the threshold remain unresolved. Two helpers for the phantom-redirect pass: `resolvePhantomCanonical(engine, sourceId, phantomSlug)` SKIPS the exact-slug step (a phantom slug `'alice'` would exact-match itself and no-op the redirect); returns the canonical only when non-null AND contains `/`. `findPrefixCandidates(engine, sourceId, token)` is a standalone SQL query returning ALL candidates across `PREFIX_EXPANSION_DIRS` (hardcoded `['people', 'companies']`) via `slug LIKE ANY($N::text[])` over patterns `dir/token` + `dir/token-%`, cap of 10 ordered by `connection_count DESC, slug ASC`. Pinned by `test/entity-resolve.test.ts` (explicit, unique, ambiguous-person, and shared-token-company cases) plus `test/phantom-redirect.test.ts` (resolvePhantomCanonical 3 cases + findPrefixCandidates 6 cases incl. multi-dir ambiguity and the `people/aliceberg`-doesn't-match-`alice` false-positive guard). diff --git a/src/core/config.ts b/src/core/config.ts index 2377fd428..89493e040 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -271,6 +271,8 @@ export interface GBrainConfig { verdict_model?: string; max_prompt_tokens?: number; max_chunks_per_transcript?: number; + subagent_timeout_ms?: number; + subagent_wait_timeout_ms?: number; }; patterns?: { lookback_days?: number; @@ -710,6 +712,12 @@ export async function loadConfigWithEngine( const n = parseInt(v, 10); return Number.isFinite(n) && n > 0 ? n : undefined; } + async function dbNum(key: string): Promise { + const v = await dbStr(key); + if (v === undefined) return undefined; + const n = Number(v); + return Number.isNaN(n) ? undefined : n; + } const dbWarnBytes = await dbInt('content_sanity.bytes_warn'); const dbBlockBytes = await dbInt('content_sanity.bytes_block'); const dbJunkEnabled = await dbBool('content_sanity.junk_patterns_enabled'); @@ -759,6 +767,8 @@ export async function loadConfigWithEngine( const dbVerdictModel = await dbStr('dream.synthesize.verdict_model'); const dbMaxPromptTokens = await dbInt('dream.synthesize.max_prompt_tokens'); const dbMaxChunksPerTranscript = await dbInt('dream.synthesize.max_chunks_per_transcript'); + const dbSubagentTimeoutMs = await dbNum('dream.synthesize.subagent_timeout_ms'); + const dbSubagentWaitTimeoutMs = await dbNum('dream.synthesize.subagent_wait_timeout_ms'); const dbLookbackDays = await dbInt('dream.patterns.lookback_days'); const dbMinEvidence = await dbInt('dream.patterns.min_evidence'); @@ -783,6 +793,12 @@ export async function loadConfigWithEngine( if (mergedSynth.max_chunks_per_transcript === undefined && dbMaxChunksPerTranscript !== undefined) { mergedSynth.max_chunks_per_transcript = dbMaxChunksPerTranscript; } + if (mergedSynth.subagent_timeout_ms === undefined && dbSubagentTimeoutMs !== undefined) { + mergedSynth.subagent_timeout_ms = dbSubagentTimeoutMs; + } + if (mergedSynth.subagent_wait_timeout_ms === undefined && dbSubagentWaitTimeoutMs !== undefined) { + mergedSynth.subagent_wait_timeout_ms = dbSubagentWaitTimeoutMs; + } if (mergedPatterns.lookback_days === undefined && dbLookbackDays !== undefined) { mergedPatterns.lookback_days = dbLookbackDays; } @@ -854,6 +870,8 @@ export const KNOWN_CONFIG_KEYS: readonly string[] = [ // subagent handler's error message tells users to `config set` this, so it // must be a known key or `config set` rejects it without --force. 'agent.use_gateway_loop', + // #2778: per-turn output-token cap for the subagent loop (default 8192). + 'agent.max_output_tokens', // DB-plane (v0.32.3 search modes + related) 'search.mode', 'search.cache.enabled', @@ -888,6 +906,8 @@ export const KNOWN_CONFIG_KEYS: readonly string[] = [ 'models.chat', 'models.eval.longmemeval', 'facts.extraction_model', + // #2113: output-token cap for the per-turn facts extractor (default 4000). + 'facts.extraction_max_tokens', // Dream cycle config 'dream.synthesize.session_corpus_dir', 'dream.synthesize.meeting_transcripts_dir', @@ -895,8 +915,14 @@ export const KNOWN_CONFIG_KEYS: readonly string[] = [ 'dream.synthesize.verdict_model', 'dream.synthesize.max_prompt_tokens', 'dream.synthesize.max_chunks_per_transcript', + 'dream.synthesize.subagent_timeout_ms', + 'dream.synthesize.subagent_wait_timeout_ms', 'dream.patterns.lookback_days', 'dream.patterns.min_evidence', + // #2782-family: patterns-phase subagent timeouts (mirror of the + // dream.synthesize.* pair from #1594). + 'dream.patterns.subagent_timeout_ms', + 'dream.patterns.subagent_wait_timeout_ms', // Emotional weight (v0.29) 'emotional_weight.high_tags', 'emotional_weight.user_holder', diff --git a/src/core/cycle/patterns.ts b/src/core/cycle/patterns.ts index 5edb28f7e..4a39779e1 100644 --- a/src/core/cycle/patterns.ts +++ b/src/core/cycle/patterns.ts @@ -96,7 +96,7 @@ export async function runPhasePatterns( }; const submitOpts: Partial = { max_stalled: 3, - timeout_ms: 30 * 60 * 1000, + timeout_ms: config.subagentTimeoutMs, }; const job = await queue.add('subagent', data as unknown as Record, submitOpts, { allowProtectedSubmit: true, @@ -105,7 +105,7 @@ export async function runPhasePatterns( let outcome: string; try { const final = await waitForCompletion(queue, job.id, { - timeoutMs: 35 * 60 * 1000, + timeoutMs: config.subagentWaitTimeoutMs, pollMs: 5 * 1000, }); outcome = final.status; @@ -126,13 +126,47 @@ export async function runPhasePatterns( // Reverse-write to fs. const reverseWriteCount = await reverseWriteRefs(engine, opts.brainDir, writtenRefs); - return ok(`${writtenRefs.length} pattern page(s) written/updated (${outcome})`, { + const details = { reflections_considered: reflections.length, patterns_written: writtenRefs.length, reverse_write_count: reverseWriteCount, child_outcome: outcome, job_id: job.id, - }); + }; + + // #2782: the phase status must reflect the child outcome. Pre-fix this + // returned status:ok even when the subagent timed out (e.g. no + // subagent-capable worker slot free for the whole wait window) and zero + // pattern pages were written — a silent no-op for days. + if (outcome !== 'complete') { + if (writtenRefs.length === 0) { + return { + phase: 'patterns', + status: 'fail', + duration_ms: 0, + summary: `pattern-detection subagent job ${job.id} ended '${outcome}'; nothing was written`, + details, + error: makeError( + outcome === 'timeout' ? 'Timeout' : 'InternalError', + `PATTERNS_CHILD_${outcome.toUpperCase()}`, + `subagent job ${job.id} outcome '${outcome}' with zero pattern pages written`, + outcome === 'timeout' + ? 'A timeout with zero writes usually means no subagent-capable worker claimed the job. Check `gbrain jobs list` and worker capacity.' + : undefined, + ), + }; + } + // Partial: the child died/timed out but some pages landed first. + return { + phase: 'patterns', + status: 'warn', + duration_ms: 0, + summary: `${writtenRefs.length} pattern page(s) written but subagent job ${job.id} ended '${outcome}'`, + details, + }; + } + + return ok(`${writtenRefs.length} pattern page(s) written/updated (${outcome})`, details); } catch (e) { return failed(makeError('InternalError', 'PATTERNS_PHASE_FAIL', e instanceof Error ? (e.message || 'patterns phase threw') : String(e))); @@ -148,6 +182,20 @@ interface PatternsConfig { lookbackDays: number; minEvidence: number; model: string; + /** #1594-family: subagent job timeout, config `dream.patterns.subagent_timeout_ms`. */ + subagentTimeoutMs: number; + /** #1594-family: waitForCompletion timeout, config `dream.patterns.subagent_wait_timeout_ms`. */ + subagentWaitTimeoutMs: number; +} + +const DEFAULT_PATTERNS_SUBAGENT_TIMEOUT_MS = 30 * 60 * 1000; +const DEFAULT_PATTERNS_SUBAGENT_WAIT_TIMEOUT_MS = 35 * 60 * 1000; + +async function getNumberConfig(engine: BrainEngine, key: string, fallback: number): Promise { + const raw = await engine.getConfig(key); + if (raw === undefined || raw === null) return fallback; + const value = Number(raw); + return Number.isNaN(value) ? fallback : value; } async function loadPatternsConfig(engine: BrainEngine): Promise { @@ -168,6 +216,12 @@ async function loadPatternsConfig(engine: BrainEngine): Promise lookbackDays: lookbackStr ? Math.max(1, parseInt(lookbackStr, 10) || 30) : 30, minEvidence: minEvidenceStr ? Math.max(1, parseInt(minEvidenceStr, 10) || 3) : 3, model, + subagentTimeoutMs: await getNumberConfig( + engine, 'dream.patterns.subagent_timeout_ms', DEFAULT_PATTERNS_SUBAGENT_TIMEOUT_MS, + ), + subagentWaitTimeoutMs: await getNumberConfig( + engine, 'dream.patterns.subagent_wait_timeout_ms', DEFAULT_PATTERNS_SUBAGENT_WAIT_TIMEOUT_MS, + ), }; } diff --git a/src/core/cycle/synthesize.ts b/src/core/cycle/synthesize.ts index 1581bc960..94564379e 100644 --- a/src/core/cycle/synthesize.ts +++ b/src/core/cycle/synthesize.ts @@ -75,6 +75,8 @@ const MIN_PROMPT_TOKENS = 100_000; const DEFAULT_MAX_CHUNKS = 24; /** Conservative default budget when model is unknown (200K × HEADROOM_RATIO). */ const UNKNOWN_MODEL_BUDGET_TOKENS = 180_000; +const DEFAULT_SUBAGENT_TIMEOUT_MS = 30 * 60 * 1000; +const DEFAULT_SUBAGENT_WAIT_TIMEOUT_MS = 35 * 60 * 1000; /** * Compute per-chunk character budget for the resolved model + config override. @@ -478,7 +480,7 @@ export async function runPhaseSynthesize( max_stalled: 3, on_child_fail: 'continue', idempotency_key, - timeout_ms: 30 * 60 * 1000, // 30 min per chunk + timeout_ms: config.subagentTimeoutMs, }; const child = await queue.add( 'subagent', @@ -499,7 +501,7 @@ export async function runPhaseSynthesize( for (const jobId of childIds) { try { const job = await waitForCompletion(queue, jobId, { - timeoutMs: 35 * 60 * 1000, + timeoutMs: config.subagentWaitTimeoutMs, pollMs: 5 * 1000, }); childOutcomes.push({ jobId, status: job.status }); @@ -593,6 +595,8 @@ interface SynthConfig { * `dream.synthesize.max_chunks_per_transcript`. */ maxChunksPerTranscript: number; + subagentTimeoutMs: number; + subagentWaitTimeoutMs: number; } async function loadSynthConfig(engine: BrainEngine): Promise { @@ -621,6 +625,16 @@ async function loadSynthConfig(engine: BrainEngine): Promise { const cooldownHoursStr = await engine.getConfig('dream.synthesize.cooldown_hours'); const maxPromptTokensStr = await engine.getConfig('dream.synthesize.max_prompt_tokens'); const maxChunksStr = await engine.getConfig('dream.synthesize.max_chunks_per_transcript'); + const subagentTimeoutMs = await getNumberConfig( + engine, + 'dream.synthesize.subagent_timeout_ms', + DEFAULT_SUBAGENT_TIMEOUT_MS, + ); + const subagentWaitTimeoutMs = await getNumberConfig( + engine, + 'dream.synthesize.subagent_wait_timeout_ms', + DEFAULT_SUBAGENT_WAIT_TIMEOUT_MS, + ); let excludePatterns: string[] = ['medical', 'therapy']; if (excludeStr) { @@ -658,9 +672,22 @@ async function loadSynthConfig(engine: BrainEngine): Promise { cooldownHours: cooldownHoursStr ? Math.max(0, parseInt(cooldownHoursStr, 10) || 12) : 12, maxPromptTokens, maxChunksPerTranscript, + subagentTimeoutMs, + subagentWaitTimeoutMs, }; } +async function getNumberConfig( + engine: BrainEngine, + key: string, + fallback: number, +): Promise { + const raw = await engine.getConfig(key); + if (raw === undefined || raw === null) return fallback; + const value = Number(raw); + return Number.isNaN(value) ? fallback : value; +} + async function checkCooldown( engine: BrainEngine, hours: number, diff --git a/src/core/facts/extract.ts b/src/core/facts/extract.ts index 1581ff4f0..0ee9930ec 100644 --- a/src/core/facts/extract.ts +++ b/src/core/facts/extract.ts @@ -67,6 +67,23 @@ export async function getFactsExtractionModel(engine?: BrainEngine): Promise`; default 4000. + */ +export const DEFAULT_EXTRACTION_MAX_TOKENS = 4000; + +export async function getFactsExtractionMaxTokens(engine?: BrainEngine): Promise { + if (!engine) return DEFAULT_EXTRACTION_MAX_TOKENS; + const raw = await engine.getConfig('facts.extraction_max_tokens').catch(() => null); + if (raw == null || raw.trim() === '') return DEFAULT_EXTRACTION_MAX_TOKENS; + const n = Number(raw); + return Number.isFinite(n) && n > 0 ? Math.floor(n) : DEFAULT_EXTRACTION_MAX_TOKENS; +} + export const ALL_EXTRACT_KINDS: readonly FactKind[] = [ 'event', 'preference', 'commitment', 'belief', 'fact', ] as const; @@ -164,24 +181,46 @@ export async function extractFactsFromTurn(input: ExtractInput): Promise\n${cleaned}\n\n\nExtract up to ${cap} facts.${ + input.entityHints && input.entityHints.length + ? ` Known entity slugs the user already mentioned: ${input.entityHints.slice(0, 5).join(', ')}.` + : '' + }`; let result: ChatResult; try { result = await chat({ - model: input.model ?? defaultModel, + model, system: EXTRACTOR_SYSTEM, - messages: [ - { - role: 'user', - content: `\n${cleaned}\n\n\nExtract up to ${cap} facts.${ - input.entityHints && input.entityHints.length - ? ` Known entity slugs the user already mentioned: ${input.entityHints.slice(0, 5).join(', ')}.` - : '' - }`, - }, - ], - maxTokens: 1500, + messages: [{ role: 'user', content: userContent }], + maxTokens, abortSignal: input.abortSignal, }); + // #2113: never checked pre-fix — a truncated response (stopReason + // 'length', e.g. reasoning tokens eating the cap on mandatory-reasoning + // models) produced unparseable JSON and silently extracted zero facts. + // Retry ONCE at double the cap, then surface the truncation loudly. + if (result.stopReason === 'length') { + process.stderr.write( + `[facts-extract] WARN: extractor output truncated at maxTokens=${maxTokens} ` + + `(model=${model}); retrying once at ${maxTokens * 2}\n`, + ); + result = await chat({ + model, + system: EXTRACTOR_SYSTEM, + messages: [{ role: 'user', content: userContent }], + maxTokens: maxTokens * 2, + abortSignal: input.abortSignal, + }); + if (result.stopReason === 'length') { + process.stderr.write( + `[facts-extract] WARN: extractor output STILL truncated at maxTokens=${maxTokens * 2} ` + + `(model=${model}); facts for this turn are likely lost. ` + + `Raise the cap: gbrain config set facts.extraction_max_tokens \n`, + ); + } + } } catch (err) { // Re-throw aborts; absorb other errors as "no extraction" — caller's // `put_page` backstop will still record the page itself. diff --git a/src/core/minions/handlers/subagent.ts b/src/core/minions/handlers/subagent.ts index 38cfdfde6..53ef86433 100644 --- a/src/core/minions/handlers/subagent.ts +++ b/src/core/minions/handlers/subagent.ts @@ -58,8 +58,29 @@ import { randomUUIDv7 } from 'bun'; const DEFAULT_MODEL = 'claude-sonnet-4-6'; const DEFAULT_MAX_TURNS = 20; +const DEFAULT_MAX_OUTPUT_TOKENS = 8192; const DEFAULT_RATE_KEY = 'anthropic:messages'; +/** + * Resolve the per-turn output-token cap (#2778). Per-job data wins, then the + * `agent.max_output_tokens` config row, then the 8192 default (was a + * hardcoded 4096 that made pages >~12KB unwritable via put_page). Invalid + * values (NaN / zero / negative) fall through to the next tier. + */ +export function resolveMaxOutputTokens( + perJob: number | undefined, + configRaw: string | null | undefined, +): number { + if (typeof perJob === 'number' && Number.isFinite(perJob) && perJob > 0) { + return Math.floor(perJob); + } + if (typeof configRaw === 'string' && configRaw.trim() !== '') { + const n = Number(configRaw); + if (Number.isFinite(n) && n > 0) return Math.floor(n); + } + return DEFAULT_MAX_OUTPUT_TOKENS; +} + /** * Resolve the rate-lease cap from the env var. * @@ -212,6 +233,11 @@ export function makeSubagentHandler(deps: SubagentDeps) { fallback: TIER_DEFAULTS.subagent, }); const maxTurns = data.max_turns ?? DEFAULT_MAX_TURNS; + // #2778: per-turn output cap — data.max_tokens → config → 8192 default. + const maxOutputTokens = resolveMaxOutputTokens( + data.max_tokens, + await engine.getConfig('agent.max_output_tokens').catch(() => null), + ); // v0.41 Approach C: systemPrompt is now built AFTER toolDefs (a few // lines below) so the renderer can splice a tool-usage preamble // listing each available tool's usage_hint. The renderer is @@ -277,6 +303,7 @@ export function makeSubagentHandler(deps: SubagentDeps) { systemPrompt, toolDefs, maxTurns, + maxOutputTokens, }); } @@ -535,7 +562,7 @@ export function makeSubagentHandler(deps: SubagentDeps) { // `model` stays qualified everywhere else (persistence, recipe // lookup at recipeIdFromModel(), capability gate). model: stripProviderPrefix(model), - max_tokens: 4096, + max_tokens: maxOutputTokens, system: [ { type: 'text', text: systemPrompt, cache_control: { type: 'ephemeral' } }, ] as any, @@ -628,7 +655,10 @@ export function makeSubagentHandler(deps: SubagentDeps) { b.type === 'tool_use', ); if (toolUses.length === 0) { - stopReason = 'end_turn'; + // #2778: an output-cap hit is NOT end_turn — the text (and possibly a + // dropped trailing tool_use block) is truncated. Surface it as its own + // stop_reason instead of silently reporting a clean end_turn. + stopReason = assistantMsg.stop_reason === 'max_tokens' ? 'max_tokens' : 'end_turn'; // Concatenate text blocks as the final answer. finalText = blocks .filter(b => b.type === 'text' && typeof b.text === 'string') @@ -741,6 +771,24 @@ export function makeSubagentHandler(deps: SubagentDeps) { } } + // #2778: a max_tokens stop with tool_use blocks means the API dropped an + // incomplete trailing block (e.g. a large put_page body that overflowed + // the cap). Tell the model so it re-issues the cut-off call (split, or + // smaller pages) instead of assuming the write happened. + if (assistantMsg.stop_reason === 'max_tokens') { + toolResults.push({ + type: 'text', + text: `[system] Your previous response hit the ${maxOutputTokens}-token output cap and was truncated; ` + + `any tool call cut off by the cap was DROPPED and did not execute. Re-issue it, splitting large content if needed.`, + } as ContentBlock); + logSubagentHeartbeat({ + job_id: ctx.id, + event: 'llm_call_completed', + turn_idx: turnIdx, + error: `stop_reason=max_tokens at cap ${maxOutputTokens}; truncation note injected`, + }); + } + // 6. Append the synthesized user turn (tool_result wrappers) to the // conversation and persist it so replay picks it up. const userIdx = nextMessageIdx++; @@ -776,6 +824,8 @@ interface GatewayRunArgs { systemPrompt: string; toolDefs: ToolDef[]; maxTurns: number; + /** #2778: per-turn output-token cap (resolved by resolveMaxOutputTokens). */ + maxOutputTokens: number; } /** @@ -793,7 +843,7 @@ interface GatewayRunArgs { * reconciler sees both shapes uniformly. */ async function runSubagentViaGateway(args: GatewayRunArgs): Promise { - const { engine, ctx, data, model, systemPrompt, toolDefs, maxTurns } = args; + const { engine, ctx, data, model, systemPrompt, toolDefs, maxTurns, maxOutputTokens } = args; // Map ToolDef → ChatToolDef (gateway shape). The gateway's chat() bridges // this to provider-specific tool definitions via the Vercel AI SDK. @@ -917,6 +967,7 @@ async function runSubagentViaGateway(args: GatewayRunArgs): Promise = new Set([ 'resolve_slugs', 'get_ingest_log', 'put_page', + // #2778: the canonical timeline-write op. Fenced exactly like put_page — + // operations.ts:enforceSubagentSlugFence confines the target slug to the + // trusted-workspace allow-list (or the wiki/agents// namespace) when + // ctx.viaSubagent=true, so a subagent can only append timeline entries to + // pages it could have written anyway. + 'add_timeline_entry', // v0.29 — Salience + Anomaly Detection. Both read-only. `get_recent_transcripts` // is intentionally NOT included: subagent calls always have ctx.remote=true, // and the v0.29 trust gate rejects remote callers — adding it here would be @@ -97,6 +103,7 @@ export const BRAIN_TOOL_USAGE_HINTS: Readonly> = { resolve_slugs: 'Resolve free-form entity names to canonical slugs (e.g. "Alice" → `people/alice-example`). Use before any tool that takes a slug if the user gave a name not a slug.', get_ingest_log: 'Read the brain ingestion log for diagnostic / verification queries.', put_page: 'Write a markdown page to the gbrain DATABASE (NOT the local filesystem). Page becomes searchable + linkable. Slug must match the agent\'s allowed namespace.', + add_timeline_entry: 'Append a dated timeline entry to an existing page (the canonical timeline write). Use over rewriting the page body when recording a dated event. Slug must match the agent\'s allowed namespace.', get_recent_salience: 'Read pages ranked by emotional + activity salience over a recency window. Use for "what\'s been on my mind lately".', find_anomalies: 'Read cohort-level activity outliers (e.g. tag-cohort or type-cohort with unusual recent volume). Use for "what\'s unusual lately".', }; diff --git a/src/core/minions/types.ts b/src/core/minions/types.ts index e935afe8b..a24d5446d 100644 --- a/src/core/minions/types.ts +++ b/src/core/minions/types.ts @@ -411,6 +411,12 @@ export interface SubagentHandlerData { model?: string; /** Max assistant turns before the loop fails with stop_reason='max_turns'. */ max_turns?: number; + /** + * Per-turn max output tokens (#2778). Resolution: this field → + * `agent.max_output_tokens` config → 8192 default. The pre-#2778 + * hardcoded 4096 made pages >~12KB unwritable via put_page. + */ + max_tokens?: number; /** * Whitelist of tool names the agent may call. MUST be a subset of the * derived registry names — invalid entries are rejected at tool-dispatch @@ -562,6 +568,7 @@ export type ContentBlock = export type SubagentStopReason = | 'end_turn' // Anthropic says end_turn and last message has no tool_use | 'max_turns' // hit max_turns budget before end_turn + | 'max_tokens' // final turn hit the output-token cap — result text is TRUNCATED (#2778) | 'refusal' // detected via stop_reason + content shape | 'error'; // unrecoverable (empty response retry exhausted, etc.) diff --git a/src/core/operations.ts b/src/core/operations.ts index 82f1d8e06..ab2d4cbf1 100644 --- a/src/core/operations.ts +++ b/src/core/operations.ts @@ -193,6 +193,39 @@ export function matchesSlugAllowList(slug: string, prefixes: readonly string[]): return false; } +/** + * Subagent slug-fence enforcement, shared by every mutating op a subagent + * can reach (put_page, add_timeline_entry). FAIL-CLOSED: `viaSubagent=true` + * enforces the check even if the dispatcher forgot to populate `subagentId`. + * + * - Trusted-workspace path (ctx.allowedSlugPrefixes set by cycle.ts under + * PROTECTED_JOB_NAMES \u2014 MCP cannot reach it): slug must match the + * allow-list globs. + * - Legacy default: slug must live under `wiki/agents//...` + * (anchored, slash-boundary \u2014 `wiki/agents/12evil/*` can't impersonate + * subagent 12). + */ +function enforceSubagentSlugFence(ctx: OperationContext, slug: string, opName: string): void { + if (ctx.viaSubagent !== true) return; + if (typeof ctx.subagentId !== 'number' || Number.isNaN(ctx.subagentId)) { + throw new OperationError('permission_denied', `${opName} via subagent requires ctx.subagentId`); + } + const allowList = ctx.allowedSlugPrefixes; + if (allowList && allowList.length > 0) { + if (!matchesSlugAllowList(slug, allowList)) { + throw new OperationError( + 'permission_denied', + `${opName} slug '${slug}' is not within the trusted-workspace allow-list (${allowList.join(', ')})` + ); + } + } else { + const prefix = `wiki/agents/${ctx.subagentId}/`; + if (!slug.startsWith(prefix) || slug.length === prefix.length) { + throw new OperationError('permission_denied', `${opName} via subagent must write under '${prefix}...'`); + } + } +} + /** * Allowlist validator for uploaded file basenames. Rejects control chars, backslashes, * RTL overrides (\u202E), leading dot (hidden files) and leading dash (CLI flag confusion). @@ -784,37 +817,9 @@ const put_page: Operation = { } // Subagent namespace enforcement (v0.15+). Runs BEFORE the dry-run - // short-circuit so preview calls surface the same rejection. Confines - // LLM-driven writes to wiki/agents//... — no leading slash - // (slug grammar rejects that), anchored, slash-boundary to defeat prefix - // collisions like `wiki/agents/12evil/*` impersonating subagent 12. - // - // FAIL-CLOSED: `viaSubagent=true` enforces the check even if the - // dispatcher forgot to populate `subagentId`. Agent-originated writes - // without an owning subagent id are rejected outright. - if (ctx.viaSubagent === true) { - if (typeof ctx.subagentId !== 'number' || Number.isNaN(ctx.subagentId)) { - throw new OperationError('permission_denied', 'put_page via subagent requires ctx.subagentId'); - } - const allowList = ctx.allowedSlugPrefixes; - if (allowList && allowList.length > 0) { - // Trusted-workspace path: explicit allow-list bounds writes. - // Set only by cycle.ts (synthesize/patterns) which submits subagent - // jobs under PROTECTED_JOB_NAMES — MCP cannot reach this branch. - if (!matchesSlugAllowList(slug, allowList)) { - throw new OperationError( - 'permission_denied', - `put_page slug '${slug}' is not within the trusted-workspace allow-list (${allowList.join(', ')})` - ); - } - } else { - // Legacy default: agent-namespace confinement. - const prefix = `wiki/agents/${ctx.subagentId}/`; - if (!slug.startsWith(prefix) || slug.length === prefix.length) { - throw new OperationError('permission_denied', `put_page via subagent must write under '${prefix}...'`); - } - } - } + // short-circuit so preview calls surface the same rejection. See + // enforceSubagentSlugFence for the fail-closed policy. + enforceSubagentSlugFence(ctx, slug, 'put_page'); if (ctx.dryRun) return { dry_run: true, action: 'put_page', slug: p.slug }; // Skip embedding when the AI gateway has no embedding provider configured. @@ -2149,6 +2154,11 @@ const add_timeline_entry: Operation = { mutating: true, scope: 'write', handler: async (ctx, p) => { + // #2778: same fail-closed slug fence as put_page. add_timeline_entry is + // subagent-allowlisted (brain-allowlist.ts), so timeline writes must be + // confined to the same namespace/allow-list as page writes. Runs before + // the dry-run short-circuit so preview calls surface the same rejection. + enforceSubagentSlugFence(ctx, p.slug as string, 'add_timeline_entry'); if (ctx.dryRun) return { dry_run: true, action: 'add_timeline_entry', slug: p.slug }; const date = p.date as string; // Reject anything that isn't a strict YYYY-MM-DD with year 1900-2199 and diff --git a/test/brain-allowlist.serial.test.ts b/test/brain-allowlist.serial.test.ts index 7ed4bcb84..60e74bba9 100644 --- a/test/brain-allowlist.serial.test.ts +++ b/test/brain-allowlist.serial.test.ts @@ -50,7 +50,10 @@ describe('BRAIN_TOOL_ALLOWLIST', () => { // have ctx.remote=true, and the v0.29 trust gate rejects remote callers. // v114 (#1941) added list_link_sources (read-only provenance discovery); // the edge-WRITE ops add_link/remove_link stay out (separate trust call). - expect(BRAIN_TOOL_ALLOWLIST.size).toBe(14); + // #2778 added add_timeline_entry (write, fenced like put_page via + // operations.ts:enforceSubagentSlugFence). + expect(BRAIN_TOOL_ALLOWLIST.size).toBe(15); + expect(BRAIN_TOOL_ALLOWLIST.has('add_timeline_entry')).toBe(true); expect(BRAIN_TOOL_ALLOWLIST.has('query')).toBe(true); expect(BRAIN_TOOL_ALLOWLIST.has('search')).toBe(true); expect(BRAIN_TOOL_ALLOWLIST.has('get_page')).toBe(true); diff --git a/test/config-set.test.ts b/test/config-set.test.ts index 9d837341d..042cc599a 100644 --- a/test/config-set.test.ts +++ b/test/config-set.test.ts @@ -30,6 +30,11 @@ describe('KNOWN_CONFIG_KEYS', () => { expect(KNOWN_CONFIG_KEYS).toContain('models.tier.subagent'); }); + test('contains the dream synthesize timeout keys (#1594)', () => { + expect(KNOWN_CONFIG_KEYS).toContain('dream.synthesize.subagent_timeout_ms'); + expect(KNOWN_CONFIG_KEYS).toContain('dream.synthesize.subagent_wait_timeout_ms'); + }); + test('contains the spend-control keys (v0.42.42.0, #2139) — no --force archaeology', () => { expect(KNOWN_CONFIG_KEYS).toContain('spend.posture'); expect(KNOWN_CONFIG_KEYS).toContain('sync.cost_gate_min_usd'); diff --git a/test/cycle-patterns-child-outcome.test.ts b/test/cycle-patterns-child-outcome.test.ts new file mode 100644 index 000000000..6ed135108 --- /dev/null +++ b/test/cycle-patterns-child-outcome.test.ts @@ -0,0 +1,103 @@ +/** + * #2782 — patterns phase status must reflect the child subagent outcome. + * + * Pre-fix, runPhasePatterns returned status:ok with child_outcome:timeout and + * zero pattern pages written (e.g. when no subagent-capable worker slot was + * free for the whole wait window) — a silent no-op for days. + * + * Drives the real phase against PGLite with the (#1594-family) configurable + * wait timeout set to 1ms and NO worker running, so the child job never + * completes: waitForCompletion throws TimeoutError → outcome 'timeout' → + * nothing written → the phase must report status 'fail', not 'ok'. + */ + +import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { PGLiteEngine } from '../src/core/pglite-engine.ts'; +import { resetPgliteState } from './helpers/reset-pglite.ts'; +import { runPhasePatterns } from '../src/core/cycle/patterns.ts'; +import { withEnv } from './helpers/with-env.ts'; + +let engine: PGLiteEngine; +let schemaVersion: string; + +beforeAll(async () => { + engine = new PGLiteEngine(); + await engine.connect({ database_url: '' }); + await engine.initSchema(); + // resetPgliteState truncates `config`, wiping the `version` row that + // MinionQueue.ensureSchema checks. Capture it so beforeEach can restore. + schemaVersion = (await engine.getConfig('version')) ?? '7'; +}, 60_000); + +afterAll(async () => { + await engine.disconnect(); +}); + +beforeEach(async () => { + await resetPgliteState(engine); + await engine.setConfig('version', schemaVersion); +}); + +async function seedReflections(): Promise { + // Enough recent reflections to clear min_evidence (default 3). + for (let i = 0; i < 3; i++) { + await engine.executeRaw( + `INSERT INTO pages (slug, type, title, compiled_truth) + VALUES ($1, 'note', $2, $3)`, + [ + `wiki/personal/reflections/2026-07-0${i + 1}-reflection`, + `Reflection ${i + 1}`, + `Recurring theme fixture number ${i + 1}.`, + ], + ); + } +} + +describe('runPhasePatterns child-outcome status (#2782)', () => { + test('child timeout with zero writes → status fail (was silent ok)', async () => { + const brainDir = mkdtempSync(join(tmpdir(), 'gbrain-patterns-outcome-')); + try { + await seedReflections(); + + // #1594-family knob: make the wait window elapse immediately. No + // minion worker runs in this test, so the child job stays queued. + await engine.setConfig('dream.patterns.subagent_wait_timeout_ms', '1'); + + const result = await withEnv({ ANTHROPIC_API_KEY: 'sk-ant-test' }, () => + runPhasePatterns(engine, { brainDir, dryRun: false }), + ); + + expect(result.status).toBe('fail'); + expect(result.details.child_outcome).toBe('timeout'); + expect(result.details.patterns_written).toBe(0); + expect(result.error?.code).toBe('PATTERNS_CHILD_TIMEOUT'); + expect(result.error?.class).toBe('Timeout'); + } finally { + rmSync(brainDir, { recursive: true, force: true }); + } + }, 60_000); + + test('dream.patterns.subagent_timeout_ms flows to the submitted job', async () => { + const brainDir = mkdtempSync(join(tmpdir(), 'gbrain-patterns-timeout-')); + try { + await seedReflections(); + await engine.setConfig('dream.patterns.subagent_timeout_ms', '600000'); + await engine.setConfig('dream.patterns.subagent_wait_timeout_ms', '1'); + + await withEnv({ ANTHROPIC_API_KEY: 'sk-ant-test' }, () => + runPhasePatterns(engine, { brainDir, dryRun: false }), + ); + + const jobs = await engine.executeRaw<{ timeout_ms: string | number | null }>( + `SELECT timeout_ms FROM minion_jobs WHERE name = 'subagent' ORDER BY id DESC LIMIT 1`, + ); + expect(jobs).toHaveLength(1); + expect(Number(jobs[0]!.timeout_ms)).toBe(600000); + } finally { + rmSync(brainDir, { recursive: true, force: true }); + } + }, 60_000); +}); diff --git a/test/cycle-synthesize-subagent-timeout.test.ts b/test/cycle-synthesize-subagent-timeout.test.ts new file mode 100644 index 000000000..2e33b4fb2 --- /dev/null +++ b/test/cycle-synthesize-subagent-timeout.test.ts @@ -0,0 +1,83 @@ +/** + * #1594 — dream synthesize subagent timeouts are config keys, not hardcoded + * 30/35-minute constants. Approach ported from PR #1596 (@ai920wisco). + */ +import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test'; +import { createHash } from 'node:crypto'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { PGLiteEngine } from '../src/core/pglite-engine.ts'; +import { resetPgliteState } from './helpers/reset-pglite.ts'; +import { runPhaseSynthesize } from '../src/core/cycle/synthesize.ts'; + +let engine: PGLiteEngine; +let schemaVersion: string; + +beforeAll(async () => { + engine = new PGLiteEngine(); + await engine.connect({ database_url: '' }); + await engine.initSchema(); + // resetPgliteState truncates `config`, wiping the `version` row that + // MinionQueue.ensureSchema checks. Capture it so beforeEach can restore. + schemaVersion = (await engine.getConfig('version')) ?? '7'; +}, 60_000); + +afterAll(async () => { + await engine.disconnect(); +}); + +beforeEach(async () => { + await resetPgliteState(engine); + await engine.setConfig('version', schemaVersion); +}); + +async function seedWorthProcessingVerdict( + filePath: string, + content: string, +): Promise { + const contentHash = createHash('sha256').update(content, 'utf8').digest('hex'); + await engine.putDreamVerdict(filePath, contentHash, { + worth_processing: true, + reasons: ['seeded for timeout config test'], + }); +} + +describe('runPhaseSynthesize subagent timeout config', () => { + test('dream.synthesize.subagent_timeout_ms flows to submitted subagent job', async () => { + const brainDir = mkdtempSync(join(tmpdir(), 'gbrain-synth-timeout-brain-')); + const corpusDir = mkdtempSync(join(tmpdir(), 'gbrain-synth-timeout-corpus-')); + + try { + await engine.setConfig('dream.synthesize.enabled', 'true'); + await engine.setConfig('dream.synthesize.session_corpus_dir', corpusDir); + await engine.setConfig('dream.synthesize.subagent_timeout_ms', '600000'); + await engine.setConfig('dream.synthesize.subagent_wait_timeout_ms', '1'); + + const filePath = join(corpusDir, '2026-05-28-dense-transcript.txt'); + const content = 'dense transcript line\n'.repeat(250); + writeFileSync(filePath, content); + await seedWorthProcessingVerdict(filePath, content); + + const result = await runPhaseSynthesize(engine, { + brainDir, + dryRun: false, + }); + + expect(result.status).toBe('ok'); + + const jobs = await engine.executeRaw<{ timeout_ms: string | number | null }>( + `SELECT timeout_ms + FROM minion_jobs + WHERE name = 'subagent' + ORDER BY id DESC + LIMIT 1`, + ); + expect(jobs).toHaveLength(1); + expect(Number(jobs[0]!.timeout_ms)).toBe(600000); + } finally { + rmSync(brainDir, { recursive: true, force: true }); + rmSync(corpusDir, { recursive: true, force: true }); + } + }, 30_000); +}); diff --git a/test/facts-extract-truncation.test.ts b/test/facts-extract-truncation.test.ts new file mode 100644 index 000000000..ec076b8ff --- /dev/null +++ b/test/facts-extract-truncation.test.ts @@ -0,0 +1,132 @@ +/** + * #2113 — facts extraction must not silently extract zero facts on truncation. + * + * Pre-fix, extractFactsFromTurn hardcoded maxTokens:1500 and never checked + * the finish reason. Mandatory-reasoning models spend thinking tokens inside + * the same cap, so the JSON payload got cut off, parse failed, and extraction + * returned [] with no signal. + * + * Post-fix: the cap is configurable (`facts.extraction_max_tokens`, default + * 4000), a stopReason:'length' response is retried once at double the cap, + * and a still-truncated retry is surfaced on stderr. + * + * Uses the gateway chat-transport test seam — no API key, no network. + */ +import { afterAll, describe, test, expect, beforeEach } from 'bun:test'; +import { + configureGateway, + resetGateway, + __setChatTransportForTests, +} from '../src/core/ai/gateway.ts'; +import type { ChatOpts, ChatResult } from '../src/core/ai/gateway.ts'; +import { + extractFactsFromTurn, + getFactsExtractionMaxTokens, + DEFAULT_EXTRACTION_MAX_TOKENS, +} from '../src/core/facts/extract.ts'; +import type { BrainEngine } from '../src/core/engine.ts'; + +beforeEach(() => { + resetGateway(); + __setChatTransportForTests(null); + configureGateway({ + chat_model: 'anthropic:claude-sonnet-4-6', + env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, + }); +}); + +// Shard hygiene (same rationale as facts-extract-silent-no-op.test.ts): +// restore the legacy 1536-d embedding pin so later fresh-schema files in +// this shard don't inherit a dimensionless gateway. +afterAll(() => { + __setChatTransportForTests(null); + configureGateway({ + embedding_model: 'openai:text-embedding-3-large', + embedding_dimensions: 1536, + env: { ...process.env }, + }); +}); + +function chatResult(text: string, stopReason: ChatResult['stopReason']): ChatResult { + return { + text, + blocks: [{ type: 'text', text }], + stopReason, + usage: { input_tokens: 1, output_tokens: 1, cache_read_tokens: 0, cache_creation_tokens: 0 }, + model: 'anthropic:claude-sonnet-4-6', + providerId: 'anthropic', + } as ChatResult; +} + +const GOOD_JSON = '{"facts":[{"fact":"user gave up alcohol","kind":"commitment",' + + '"entity":null,"confidence":1.0,"notability":"high",' + + '"metric":null,"value":null,"unit":null,"period":null}]}'; + +describe('getFactsExtractionMaxTokens (#2113)', () => { + test('defaults to 4000 without an engine', async () => { + expect(await getFactsExtractionMaxTokens()).toBe(DEFAULT_EXTRACTION_MAX_TOKENS); + expect(DEFAULT_EXTRACTION_MAX_TOKENS).toBe(4000); + }); + + test('reads facts.extraction_max_tokens from the engine', async () => { + const engine = { getConfig: async () => '9000' } as unknown as BrainEngine; + expect(await getFactsExtractionMaxTokens(engine)).toBe(9000); + }); + + test('invalid config values fall back to the default', async () => { + for (const bad of ['garbage', '0', '-5', '']) { + const engine = { getConfig: async () => bad } as unknown as BrainEngine; + expect(await getFactsExtractionMaxTokens(engine)).toBe(DEFAULT_EXTRACTION_MAX_TOKENS); + } + }); +}); + +describe('extractFactsFromTurn truncation handling (#2113)', () => { + test('default call carries maxTokens=4000 (was hardcoded 1500)', async () => { + const seen: ChatOpts[] = []; + __setChatTransportForTests(async (opts) => { + seen.push(opts); + return chatResult(GOOD_JSON, 'end'); + }); + const facts = await extractFactsFromTurn({ + turnText: 'I gave up alcohol.', + source: 'test:truncation', + }); + expect(seen).toHaveLength(1); + expect(seen[0]!.maxTokens).toBe(4000); + expect(facts).toHaveLength(1); + }); + + test("stopReason 'length' retries ONCE at double the cap and recovers the facts", async () => { + const seen: ChatOpts[] = []; + __setChatTransportForTests(async (opts) => { + seen.push(opts); + // First call: truncated garbage. Retry: full JSON. + return seen.length === 1 + ? chatResult('{"facts":[{"fact":"user gave up alco', 'length') + : chatResult(GOOD_JSON, 'end'); + }); + const facts = await extractFactsFromTurn({ + turnText: 'I gave up alcohol.', + source: 'test:truncation', + }); + expect(seen).toHaveLength(2); + expect(seen[1]!.maxTokens).toBe(seen[0]!.maxTokens! * 2); + expect(facts).toHaveLength(1); + expect(facts[0]!.fact).toContain('alcohol'); + }); + + test('still-truncated retry does not retry again (bounded at one retry)', async () => { + let calls = 0; + __setChatTransportForTests(async () => { + calls++; + return chatResult('{"facts":[{"fac', 'length'); + }); + const facts = await extractFactsFromTurn({ + turnText: 'I gave up alcohol.', + source: 'test:truncation', + }); + expect(calls).toBe(2); + expect(facts).toEqual([]); + }); +}); diff --git a/test/loadConfig-merge.test.ts b/test/loadConfig-merge.test.ts index 55c625339..7a8ab08af 100644 --- a/test/loadConfig-merge.test.ts +++ b/test/loadConfig-merge.test.ts @@ -178,7 +178,7 @@ describe('loadConfigWithEngine (Phase 4 / F3)', () => { // dream.* — adding env shadows is a separate PR (out of scope for the // fix wave). These tests pin that contract. describe('dream.* DB-plane merge (v0.41.2.1)', () => { - test('DB value fills in for all 5 dream.synthesize.* keys when base unset', async () => { + test('DB value fills in for dream.synthesize.* keys when base unset', async () => { const base: GBrainConfig = { engine: 'pglite' }; const engine = makeEngine({ 'dream.synthesize.session_corpus_dir': '/tmp/sessions', @@ -186,6 +186,8 @@ describe('loadConfigWithEngine (Phase 4 / F3)', () => { 'dream.synthesize.verdict_model': 'anthropic:claude-haiku-4-5', 'dream.synthesize.max_prompt_tokens': '180000', 'dream.synthesize.max_chunks_per_transcript': '32', + 'dream.synthesize.subagent_timeout_ms': '600000', + 'dream.synthesize.subagent_wait_timeout_ms': '900000', }); const merged = await loadConfigWithEngine(engine, base); expect(merged?.dream?.synthesize?.session_corpus_dir).toBe('/tmp/sessions'); @@ -193,6 +195,8 @@ describe('loadConfigWithEngine (Phase 4 / F3)', () => { expect(merged?.dream?.synthesize?.verdict_model).toBe('anthropic:claude-haiku-4-5'); expect(merged?.dream?.synthesize?.max_prompt_tokens).toBe(180000); expect(merged?.dream?.synthesize?.max_chunks_per_transcript).toBe(32); + expect(merged?.dream?.synthesize?.subagent_timeout_ms).toBe(600000); + expect(merged?.dream?.synthesize?.subagent_wait_timeout_ms).toBe(900000); }); test('DB value fills in for both dream.patterns.* keys when base unset', async () => { diff --git a/test/subagent-handler.test.ts b/test/subagent-handler.test.ts index 145cfc3d8..3b50a7122 100644 --- a/test/subagent-handler.test.ts +++ b/test/subagent-handler.test.ts @@ -596,3 +596,115 @@ describe('makeSubagentHandler default client construction', () => { expect(result.result).toBe('ok'); }); }); + +// ── #2778: per-turn output-token cap + max_tokens stop handling ───── + +import { resolveMaxOutputTokens } from '../src/core/minions/handlers/subagent.ts'; + +describe('resolveMaxOutputTokens (#2778)', () => { + test('defaults to 8192 when nothing set', () => { + expect(resolveMaxOutputTokens(undefined, null)).toBe(8192); + expect(resolveMaxOutputTokens(undefined, undefined)).toBe(8192); + }); + + test('per-job value wins over config', () => { + expect(resolveMaxOutputTokens(2048, '5000')).toBe(2048); + }); + + test('config value used when per-job unset', () => { + expect(resolveMaxOutputTokens(undefined, '5000')).toBe(5000); + }); + + test('invalid values fall through to next tier', () => { + expect(resolveMaxOutputTokens(0, '5000')).toBe(5000); + expect(resolveMaxOutputTokens(-1, null)).toBe(8192); + expect(resolveMaxOutputTokens(Number.NaN, 'garbage')).toBe(8192); + expect(resolveMaxOutputTokens(undefined, '')).toBe(8192); + expect(resolveMaxOutputTokens(undefined, '0')).toBe(8192); + }); +}); + +describe('subagent handler output-token cap (#2778)', () => { + test('default: SDK call carries max_tokens=8192 (was hardcoded 4096)', async () => { + const client = new FakeMessagesClient([ + { content: [{ type: 'text', text: 'ok' }] as any, stop_reason: 'end_turn' }, + ]); + const handler = makeSubagentHandler({ engine, client, toolRegistry: [] }); + const ctx = await makeCtx({ prompt: 'hi' }); + await handler(ctx); + expect(client.calls[0]!.max_tokens).toBe(8192); + }); + + test('data.max_tokens flows to the SDK call', async () => { + const client = new FakeMessagesClient([ + { content: [{ type: 'text', text: 'ok' }] as any, stop_reason: 'end_turn' }, + ]); + const handler = makeSubagentHandler({ engine, client, toolRegistry: [] }); + const ctx = await makeCtx({ prompt: 'hi', max_tokens: 2048 }); + await handler(ctx); + expect(client.calls[0]!.max_tokens).toBe(2048); + }); + + test('agent.max_output_tokens config flows to the SDK call', async () => { + await engine.setConfig('agent.max_output_tokens', '5000'); + try { + const client = new FakeMessagesClient([ + { content: [{ type: 'text', text: 'ok' }] as any, stop_reason: 'end_turn' }, + ]); + const handler = makeSubagentHandler({ engine, client, toolRegistry: [] }); + const ctx = await makeCtx({ prompt: 'hi' }); + await handler(ctx); + expect(client.calls[0]!.max_tokens).toBe(5000); + } finally { + await engine.executeRaw(`DELETE FROM config WHERE key = 'agent.max_output_tokens'`); + } + }); + + test('final turn hitting the cap surfaces stop_reason=max_tokens, not a silent end_turn', async () => { + const client = new FakeMessagesClient([ + { content: [{ type: 'text', text: 'truncated tex' }] as any, stop_reason: 'max_tokens' }, + ]); + const handler = makeSubagentHandler({ engine, client, toolRegistry: [] }); + const ctx = await makeCtx({ prompt: 'hi' }); + const result = await handler(ctx); + expect(result.stop_reason).toBe('max_tokens'); + expect(result.result).toBe('truncated tex'); + }); + + test('max_tokens stop with tool_use: truncation note injected so the model re-issues the dropped call', async () => { + const tool = makeEchoTool(); + const client = new FakeMessagesClient([ + { + // A complete tool_use survived, but the turn stopped on max_tokens — + // the API dropped whatever came after (e.g. a big put_page call). + content: [{ type: 'tool_use', id: 'tu_1', name: 'echo', input: { value: 'v1' } } as any], + stop_reason: 'max_tokens' as any, + }, + { content: [{ type: 'text', text: 'recovered' }] as any, stop_reason: 'end_turn' }, + ]); + const handler = makeSubagentHandler({ engine, client, toolRegistry: [tool] }); + const ctx = await makeCtx({ prompt: 'go' }); + + const result = await handler(ctx); + expect(result.stop_reason).toBe('end_turn'); + expect(result.result).toBe('recovered'); + + // The synthesized user turn (persisted + fed to the second call) must + // carry the truncation note alongside the tool_result. Assert on the + // persisted row — client.calls[].messages is the live array the loop + // keeps mutating, so positional checks there are unreliable. + const rows = await engine.executeRaw<{ content_blocks: unknown }>( + `SELECT content_blocks FROM subagent_messages + WHERE job_id = $1 AND role = 'user' AND message_idx > 0 + ORDER BY message_idx ASC`, + [ctx.id], + ); + expect(rows.length).toBe(1); + const blocks = (typeof rows[0]!.content_blocks === 'string' + ? JSON.parse(rows[0]!.content_blocks as string) + : rows[0]!.content_blocks) as Array<{ type: string; text?: string }>; + expect(blocks.some(b => b.type === 'tool_result')).toBe(true); + const texts = blocks.filter(b => b.type === 'text').map(b => b.text ?? ''); + expect(texts.some(t => t.includes('truncated') && t.includes('DROPPED'))).toBe(true); + }); +}); diff --git a/test/timeline-entry-subagent-fence.test.ts b/test/timeline-entry-subagent-fence.test.ts new file mode 100644 index 000000000..ac8506bc8 --- /dev/null +++ b/test/timeline-entry-subagent-fence.test.ts @@ -0,0 +1,102 @@ +/** + * #2778 — add_timeline_entry subagent slug fence. + * + * add_timeline_entry joined the subagent brain-tool allowlist, so it must be + * confined exactly like put_page: when ctx.viaSubagent=true the target slug + * must match the trusted-workspace allow-list (when set) or the legacy + * wiki/agents// namespace, fail-closed on a missing subagentId. + * Non-subagent callers (CLI, plain MCP) are unchanged. + * + * Uses dryRun ctxs — the fence runs BEFORE the dry-run short-circuit, so no + * engine is needed (same pattern as test/put-page-namespace.test.ts). + */ + +import { describe, test, expect } from 'bun:test'; +import { operations, OperationError } from '../src/core/operations.ts'; +import type { OperationContext, Operation } from '../src/core/operations.ts'; +import type { BrainEngine } from '../src/core/engine.ts'; + +const add_timeline_entry = operations.find(o => o.name === 'add_timeline_entry') as Operation; +if (!add_timeline_entry) throw new Error('add_timeline_entry op missing'); + +const ENTRY = { date: '2026-07-01', summary: 'test entry' }; + +function makeCtx(overrides: Partial = {}): OperationContext { + const engine = {} as BrainEngine; // dry_run short-circuits before touching the engine + return { + engine, + config: { engine: 'postgres' } as any, + logger: { info: () => {}, warn: () => {}, error: () => {} }, + dryRun: true, + remote: true, + sourceId: 'default', + ...overrides, + }; +} + +describe('add_timeline_entry subagent fence (#2778)', () => { + describe('regression: non-subagent callers unchanged', () => { + test('local CLI write (viaSubagent undefined) accepts arbitrary slug', async () => { + const ctx = makeCtx({ remote: false }); + const result = await add_timeline_entry.handler(ctx, { slug: 'people/alice-example', ...ENTRY }); + expect(result).toMatchObject({ dry_run: true, action: 'add_timeline_entry', slug: 'people/alice-example' }); + }); + + test('MCP write (remote=true, viaSubagent=undefined) accepts arbitrary slug', async () => { + const ctx = makeCtx({ remote: true }); + const result = await add_timeline_entry.handler(ctx, { slug: 'companies/acme-example', ...ENTRY }); + expect(result).toMatchObject({ dry_run: true }); + }); + + test('viaSubagent=false is the same as unset', async () => { + const ctx = makeCtx({ viaSubagent: false, subagentId: 42 }); + const result = await add_timeline_entry.handler(ctx, { slug: 'anything/goes', ...ENTRY }); + expect(result).toMatchObject({ dry_run: true }); + }); + }); + + describe('legacy namespace confinement', () => { + test('accepts wiki/agents// prefix', async () => { + const ctx = makeCtx({ viaSubagent: true, subagentId: 42 }); + const result = await add_timeline_entry.handler(ctx, { slug: 'wiki/agents/42/notes', ...ENTRY }); + expect(result).toMatchObject({ dry_run: true }); + }); + + test('rejects a slug outside the namespace', async () => { + const ctx = makeCtx({ viaSubagent: true, subagentId: 42 }); + const p = add_timeline_entry.handler(ctx, { slug: 'people/alice-example', ...ENTRY }); + await expect(p).rejects.toBeInstanceOf(OperationError); + await expect(p).rejects.toThrow(/add_timeline_entry/); + }); + + test('rejects prefix-collision attempt (wiki/agents/12evil/* with subagentId=12)', async () => { + const ctx = makeCtx({ viaSubagent: true, subagentId: 12 }); + const p = add_timeline_entry.handler(ctx, { slug: 'wiki/agents/12evil/foo', ...ENTRY }); + await expect(p).rejects.toBeInstanceOf(OperationError); + }); + + test('FAIL-CLOSED: viaSubagent=true with undefined subagentId rejects any slug', async () => { + const ctx = makeCtx({ viaSubagent: true }); + const p = add_timeline_entry.handler(ctx, { slug: 'wiki/agents/42/foo', ...ENTRY }); + await expect(p).rejects.toBeInstanceOf(OperationError); + await expect(p).rejects.toThrow(/subagentId/); + }); + }); + + describe('trusted-workspace allow-list', () => { + const allow = ['wiki/personal/patterns/*', 'wiki/originals/*']; + + test('accepts a slug inside the allow-list', async () => { + const ctx = makeCtx({ viaSubagent: true, subagentId: 7, allowedSlugPrefixes: allow }); + const result = await add_timeline_entry.handler(ctx, { slug: 'wiki/personal/patterns/topic-x', ...ENTRY }); + expect(result).toMatchObject({ dry_run: true }); + }); + + test('rejects a slug outside the allow-list (even inside the legacy namespace)', async () => { + const ctx = makeCtx({ viaSubagent: true, subagentId: 7, allowedSlugPrefixes: allow }); + const p = add_timeline_entry.handler(ctx, { slug: 'wiki/agents/7/notes', ...ENTRY }); + await expect(p).rejects.toBeInstanceOf(OperationError); + await expect(p).rejects.toThrow(/allow-list/); + }); + }); +});