From e861b92da7c4fde3dde8058c3b2df396b1413a4b Mon Sep 17 00:00:00 2001 From: "Benjamin D. Smith" Date: Wed, 22 Jul 2026 09:50:43 +1000 Subject: [PATCH] feat(synopsis): tail-truncate documentText for small-model chat handlers (#1427) * feat(synopsis): tail-truncate documentText for small-model chat handlers Small local chat models (Gemma 4 E2B, Qwen3 4B) get dramatically slower on long contexts even at 131K declared windows. A 73K-char page synopsis on Gemma 4 E2B takes 60-120s, exceeding the worker's default 30s `lockDuration` and tripping `lock-lost` errors. Add `SYNOPSIS_DOC_MAX_CHARS` env-overridable cap (default 32768 chars, ~8K tokens) applied in `buildUserPrompt`. Truncate the TAIL so the head (title, frontmatter, intro) preserves the document-level anchor the synopsis needs. Anthropic Haiku is unaffected at this cap; bump via `GBRAIN_SYNOPSIS_DOC_MAX_CHARS` for frontier models that want richer document anchoring. Belt-and-suspenders companion to commit 0aaff691 (--lock-duration flag on the worker). Combined: bumping lock TTL gives the handler more time, AND truncating doc cap makes the handler complete faster. Either alone helps; both together get the synopsis backfill running reliably on small local LLMs. Verified: real 1383-chunk personal brain backfill at GBRAIN_SYNOPSIS_MODEL=lmstudio:google/gemma-4-e2b + GBRAIN_SYNOPSIS_DOC_MAX_CHARS=16384 + `gbrain jobs work --concurrency 4 --lock-duration 300000` transitions from "lock-lost on every transcript page" to "no deaths, no stalls, steady throughput." RECOVERY REBUILD 2026-05-26 of original ac213aa6. * fix: fold SYNOPSIS_DOC_MAX_CHARS into corpus_generation hash Codex review of #1427 flagged that changing GBRAIN_SYNOPSIS_DOC_MAX_CHARS shifts the synopsis prompt + downstream embeddings for long documents but was NOT folded into the computeCorpusGeneration hash. Pages re-embedded with a different cap would retain the same corpus_generation, defeating the v0.40.3.0 D27 P1-5 cache invalidation contract. Three changes: 1. Export SYNOPSIS_DOC_MAX_CHARS from src/core/page-summary.ts 2. computeCorpusGeneration accepts optional synopsisDocMaxChars param. When set, folded into hash via '|doc_cap='. Omitted for non-synopsis modes (title / none don't consult the cap) so existing pre-PR caches stay valid for those. 3. Service-layer call sites (2 in contextual-retrieval-service.ts) pass SYNOPSIS_DOC_MAX_CHARS when attemptMode/resolution.mode is per_chunk_synopsis, undefined otherwise. 4. import-file.ts inline path passes undefined (per_chunk_synopsis refused upstream there). One-time effect: per_chunk_synopsis pages re-embedded post-PR get a NEW corpus_generation including the cap. v0.40.3.0 query_cache.page_generations contract auto-invalidates cached query results on first re-embed. Future cap changes track correctly. Addresses codex review P2 on PR #1427. --------- Co-authored-by: Time Attakc <89218912+time-attack@users.noreply.github.com> --- src/core/contextual-retrieval-service.ts | 27 +++++++++++++---- src/core/import-file.ts | 5 ++++ src/core/page-summary.ts | 37 +++++++++++++++++++++++- 3 files changed, 63 insertions(+), 6 deletions(-) diff --git a/src/core/contextual-retrieval-service.ts b/src/core/contextual-retrieval-service.ts index 0e22b7746..a65c8e074 100644 --- a/src/core/contextual-retrieval-service.ts +++ b/src/core/contextual-retrieval-service.ts @@ -54,6 +54,7 @@ import { import { generatePerChunkSynopsis, SYNOPSIS_PROMPT_VERSION, + SYNOPSIS_DOC_MAX_CHARS, type GeneratePerChunkSynopsisResult, } from './page-summary.ts'; import { @@ -103,8 +104,17 @@ function getEmbeddingModelTag(): string { export function computeCorpusGeneration(args: { crMode: CRMode; haikuModel: string; + /** + * Resolved `SYNOPSIS_DOC_MAX_CHARS` for per_chunk_synopsis runs. When + * present, folded into the hash so changes to + * `GBRAIN_SYNOPSIS_DOC_MAX_CHARS` invalidate the prior cache cleanly. + * Omit for `crMode !== 'per_chunk_synopsis'` — title / none modes + * don't consult the cap and the field stays out of the hash for + * back-compat with pre-cap embeddings. + */ + synopsisDocMaxChars?: number; }): string { - return createHash('sha256') + const h = createHash('sha256') .update(args.crMode) .update('|') .update(String(SYNOPSIS_PROMPT_VERSION)) @@ -113,9 +123,11 @@ export function computeCorpusGeneration(args: { .update('|') .update(String(TITLE_WRAPPER_VERSION)) .update('|') - .update(getEmbeddingModelTag()) - .digest('hex') - .slice(0, 16); + .update(getEmbeddingModelTag()); + if (args.synopsisDocMaxChars !== undefined) { + h.update('|doc_cap=').update(String(args.synopsisDocMaxChars)); + } + return h.digest('hex').slice(0, 16); } /** @@ -253,7 +265,11 @@ export async function reembedPageWithContextualRetrieval( args.pageSlug, args.sourceId, resolution.mode, - computeCorpusGeneration({ crMode: resolution.mode, haikuModel: args.haikuModel ?? DEFAULT_HAIKU_MODEL }), + computeCorpusGeneration({ + crMode: resolution.mode, + haikuModel: args.haikuModel ?? DEFAULT_HAIKU_MODEL, + synopsisDocMaxChars: resolution.mode === 'per_chunk_synopsis' ? SYNOPSIS_DOC_MAX_CHARS : undefined, + }), ); return { kind: 'skipped', reason: 'no_chunks' }; } @@ -282,6 +298,7 @@ export async function reembedPageWithContextualRetrieval( const corpus_generation = computeCorpusGeneration({ crMode: attemptMode, haikuModel, + synopsisDocMaxChars: attemptMode === 'per_chunk_synopsis' ? SYNOPSIS_DOC_MAX_CHARS : undefined, }); // ── PHASE 2: single DB transaction ─────────────────────────── diff --git a/src/core/import-file.ts b/src/core/import-file.ts index 4e7722abb..f988ee1cf 100644 --- a/src/core/import-file.ts +++ b/src/core/import-file.ts @@ -733,6 +733,11 @@ export async function importFromContent( : computeCorpusGeneration({ crMode: effectiveCRMode, haikuModel: 'anthropic:claude-haiku-4-5-20251001', + // Inline import-file path never uses per_chunk_synopsis (refuses + // upstream); pass undefined so the doc-cap field stays out of + // the hash here. Per_chunk_synopsis runs through the Minion + // backfill handler which threads SYNOPSIS_DOC_MAX_CHARS through + // the service layer. }); // Transaction wraps all DB writes. Every per-page tx call carries the diff --git a/src/core/page-summary.ts b/src/core/page-summary.ts index c4d7deedc..78e04d41c 100644 --- a/src/core/page-summary.ts +++ b/src/core/page-summary.ts @@ -44,6 +44,33 @@ const HAIKU_MAX_TOKENS = 200; /** Default model when caller doesn't override. Resolves through the gateway. */ const DEFAULT_SYNOPSIS_MODEL = 'anthropic:claude-haiku-4-5-20251001'; +/** + * Hard cap on `documentText` length (chars) before send. + * + * 2026-05-25 fix wave: small local chat models (Gemma 4 E2B, Qwen3 4B) get + * dramatically slower on long contexts even with 131K-token windows declared. + * A 73K-char page synopsis on Gemma 4 E2B takes 60-120s, exceeding the + * worker's default 30s `lockDuration` and tripping `lock-lost` errors. + * + * Truncate to a budget that fits a small model's effective throughput while + * preserving enough document context for the synopsis to be useful. Truncates + * the TAIL because the head (title, frontmatter, intro) carries the + * document-level anchor the synopsis needs. + * + * Override per workload via `GBRAIN_SYNOPSIS_DOC_MAX_CHARS`. Default 32768 + * (~8K tokens at 4 chars/tok) keeps small-model synopsis under ~30s. + * Anthropic Haiku is unaffected at this cap; bump higher when running + * frontier models if you want richer document anchoring. + */ +export const SYNOPSIS_DOC_MAX_CHARS = (() => { + const env = process.env.GBRAIN_SYNOPSIS_DOC_MAX_CHARS; + if (env && /^\d+$/.test(env)) { + const n = parseInt(env, 10); + if (n >= 512 && n <= 1_048_576) return n; + } + return 32768; +})(); + /** * Synopsis prompt version. Folded into corpus_generation so prompt edits * invalidate prior embeddings via the v0.40.3.0 query_cache.page_generations @@ -188,11 +215,19 @@ function buildUserPrompt( documentText: string, chunkText: string, ): string { + // Tail-truncate `documentText` to `SYNOPSIS_DOC_MAX_CHARS` so small local + // chat models don't stall on >100KB pages. Head preserved (title block, + // frontmatter, intro paragraphs carry the document-level anchor). + let trimmedDoc = documentText; + if (documentText.length > SYNOPSIS_DOC_MAX_CHARS) { + trimmedDoc = documentText.slice(0, SYNOPSIS_DOC_MAX_CHARS) + + `\n\n[... ${documentText.length - SYNOPSIS_DOC_MAX_CHARS} chars truncated for synopsis budget ...]`; + } return [ `${pageTitle}`, '', '', - documentText, + trimmedDoc, '', '', '',