diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6a80e4b77..db63724f8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -206,7 +206,11 @@ jobs: needs: cache-check if: needs.cache-check.outputs.hit != 'true' runs-on: ubuntu-latest - timeout-minutes: 15 + # 22, not 15: under parallel PR load the PGLite WASM cold-starts stretch a + # shard past 15 min while every test is still passing — the timeout then + # cancels the job and the test-status gate reads it as a failure. 13 runs + # died this way on 2026-07-21/22 alone. + timeout-minutes: 22 strategy: fail-fast: false matrix: diff --git a/src/commands/autopilot.ts b/src/commands/autopilot.ts index 82979fb12..67a7c20bf 100644 --- a/src/commands/autopilot.ts +++ b/src/commands/autopilot.ts @@ -1186,6 +1186,15 @@ function writeWrapperScript(repoPath: string): string { # OPENAI/ANTHROPIC keys exported in zshenv reach autopilot. [ -f ~/.zshenv ] && source ~/.zshenv 2>/dev/null source ~/.zshrc 2>/dev/null || source ~/.bashrc 2>/dev/null || true +# Belt-and-suspenders PATH fix. ~/.bashrc ships with a non-interactive guard +# (\`case $- in *i*) ;; *) return;; esac\`) that exits early when launched from +# cron/systemd/launchd — so its PATH exports never reach this subprocess. +# Without bun on PATH, the exec'd gbrain (a \`#!/usr/bin/env bun\` script) fails +# silently with "env: bun: No such file or directory" and leaves a stale +# lockfile that blocks every subsequent tick. Prepending ~/.bun/bin here +# keeps the wrapper self-contained regardless of which init file the OS +# loaded. +export PATH="$HOME/.bun/bin:$PATH" exec '${safeGbrainPath}' autopilot --repo '${safeRepoPath}' `; writeFileSync(wrapperPath, wrapper, { mode: 0o755 }); diff --git a/src/commands/code-def.ts b/src/commands/code-def.ts index 7cb3690a2..26573cae8 100644 --- a/src/commands/code-def.ts +++ b/src/commands/code-def.ts @@ -37,9 +37,19 @@ export async function findCodeDef( // trigger) are first-class definitions in the SQL sense. The chunker's // normalizeSymbolType maps create_table → 'table' etc, so adding the SQL // kinds here is what makes `gbrain code-def users` work against SQL. + // Method-level + member definitions. normalizeSymbolType only canonicalizes + // some node types; the rest fall through `type.replace(/_/g, ' ')`, so + // tree-sitter's method_declaration → 'method declaration', struct_specifier → + // 'struct specifier', protocol_declaration → 'protocol declaration', etc. + // Without these, code-def is blind to every method, constructor, field, C + // struct, and Swift protocol — which is most of an OO codebase. The plain + // 'struct' entry above never matched for the same reason (C emits the + // 'struct specifier' fallback form). const DEF_TYPES = [ 'function', 'class', 'interface', 'type', 'enum', 'struct', 'trait', 'module', 'contract', 'table', 'view', 'index', 'procedure', 'schema', 'database', 'trigger', + 'method declaration', 'method definition', 'constructor declaration', + 'field declaration', 'field definition', 'struct specifier', 'protocol declaration', ]; const params: unknown[] = [symbol, limit]; let whereLang = ''; diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index c9457d318..e57025188 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -4963,8 +4963,7 @@ export async function buildChecks( message: `${unmatched}/${sample.length} conversation pages (${unmatchedPct.toFixed(1)}%) match NO built-in pattern. ` + `Breakdown: ${breakdown}. ` + - `Investigate: gbrain conversation-parser scan | ` + - `Enable LLM fallback (opt-in): gbrain config set conversation_parser.llm_fallback_enabled true`, + `Investigate: gbrain conversation-parser scan `, }); } else { checks.push({ @@ -7778,27 +7777,71 @@ export async function runRemediationPlan( return; } - // Human output - console.log(`Brain score: ${plan.brain_score_current}/100 → target ${targetScore}`); + for (const line of renderRemediationPlanLines(plan, targetScore)) { + console.log(line); + } +} + +/** + * Human-render the remediation plan into a sequence of console lines. + * Exported for unit-test access — `runRemediationPlan` consumes it + * verbatim and only adds the JSON-mode short-circuit. + * + * Gating the "at target" line on `brain_score_current >= targetScore` + * is load-bearing: when the plan is empty AND the target is unreachable, + * the prior shape printed both "Target unreachable: …" and "Brain is at + * target" back-to-back, which contradicted itself and hid the real next + * step (manual prereq config to lift `max_reachable_score`). + */ +export function renderRemediationPlanLines( + plan: RemediationPlanShape, + targetScore: number, +): string[] { + const lines: string[] = []; + lines.push(`Brain score: ${plan.brain_score_current}/100 → target ${targetScore}`); if (plan.target_unreachable) { - console.log(`Target unreachable: max with autonomous remediation is ${plan.max_reachable_score}/100.`); + lines.push(`Target unreachable: max with autonomous remediation is ${plan.max_reachable_score}/100.`); } if (plan.plan.length === 0) { - console.log('No remediations needed. Brain is at target.'); + if (plan.brain_score_current >= targetScore) { + lines.push('No remediations needed. Brain is at target.'); + } + // When brain_score < targetScore and plan is empty, the unreachable + // line (if applicable) is the user-facing explanation; the blocked- + // checks block below surfaces the manual gap. Don't follow with a + // misleading "at target" claim. } else { - console.log(`Plan: ${plan.plan.length} step(s), est ${plan.est_total_seconds}s, est $${plan.est_total_usd_cost.toFixed(2)}`); + lines.push(`Plan: ${plan.plan.length} step(s), est ${plan.est_total_seconds}s, est $${plan.est_total_usd_cost.toFixed(2)}`); for (const step of plan.plan) { const protectedMark = step.protected ? ' [PROTECTED]' : ''; const costMark = step.est_usd_cost ? ` ($${step.est_usd_cost.toFixed(2)})` : ''; - console.log(` ${step.step}. [${step.severity}] ${step.job}${protectedMark} — ${step.rationale}${costMark}`); + lines.push(` ${step.step}. [${step.severity}] ${step.job}${protectedMark} — ${step.rationale}${costMark}`); } } if (plan.blocked.length > 0) { - console.log(`\nBlocked checks (prereq missing):`); + lines.push(`\nBlocked checks (prereq missing):`); for (const b of plan.blocked) { - console.log(` - ${b.check}: ${b.reason}`); + lines.push(` - ${b.check}: ${b.reason}`); } } + return lines; +} + +interface RemediationPlanShape { + brain_score_current: number; + target_unreachable: boolean; + max_reachable_score: number; + plan: Array<{ + step: number; + severity: string; + job: string; + protected?: boolean; + est_usd_cost?: number; + rationale: string; + }>; + est_total_seconds: number; + est_total_usd_cost: number; + blocked: Array<{ check: string; reason: string }>; } /** diff --git a/src/commands/extract.ts b/src/commands/extract.ts index db232ec51..98176e317 100644 --- a/src/commands/extract.ts +++ b/src/commands/extract.ts @@ -1743,7 +1743,18 @@ export async function extractStaleFromDB( // `page.updated_at.toISOString()` — the JS Date is ms-truncated, so the // µs-precision DB updated_at stayed strictly greater and the page never // cleared on Postgres. Stamping the exact value makes them equal. - processedRefs.push({ slug: page.slug, source_id: page.source_id, extractedAt: page.updated_at_iso }); + // + // BUT the stamp must also clear the version-staleness clause + // (`links_extracted_at < versionTs`). A page whose updated_at predates + // versionTs would otherwise be stamped below the threshold and read as + // stale forever — a permanent re-extract loop that never clears the lag. + // GREATEST(updated_at, versionTs) preserves the race semantics (a real + // future edit advances updated_at > versionTs >= stamp → re-extracts) + // while lifting old pages to the threshold so they clear. + const stampIso = page.updated_at.getTime() >= Date.parse(versionTs) + ? page.updated_at_iso + : versionTs; + processedRefs.push({ slug: page.slug, source_id: page.source_id, extractedAt: stampIso }); } // Flush NON-swallowing (CDX-4): a throw here propagates out of the sweep so diff --git a/src/commands/integrity.ts b/src/commands/integrity.ts index c883c1b48..42bfdef3d 100644 --- a/src/commands/integrity.ts +++ b/src/commands/integrity.ts @@ -98,8 +98,17 @@ export function findBareTweetHits(compiledTruth: string, slug: string): BareTwee } // If the line already contains a tweet URL, it's cited — skip if (URL_NEARBY_RE.test(line)) continue; + // If the line carries an explicit source citation (e.g. + // "[Source: X, @handle, 2026-05-28]"), it's already attributed — skip. + // Catches instructional/example lines in recipe docs that demonstrate + // the CORRECT citation format. (v0.42.x) + if (/\[\s*source:/i.test(line)) continue; + // Strip inline-code spans (`...`) before matching: phrases shown as + // inline-code templates in docs are examples, not bare claims. The + // fenced-code skip above only covers ``` blocks, not inline backticks. + const lineForMatch = line.replace(/`[^`]*`/g, ''); for (const re of BARE_TWEET_PHRASES) { - const m = line.match(re); + const m = lineForMatch.match(re); if (m) { hits.push({ slug, line: i + 1, rawLine: line.trim(), phrase: m[0] }); break; // one finding per line is enough diff --git a/src/commands/jobs.ts b/src/commands/jobs.ts index 31aa24e14..c5bbe55ad 100644 --- a/src/commands/jobs.ts +++ b/src/commands/jobs.ts @@ -1664,7 +1664,13 @@ export async function registerBuiltinHandlers( worker.register('backlinks', async (job) => { const { runBacklinksCore } = await import('./backlinks.ts'); - const action: 'check' | 'fix' = job.data.action === 'check' ? 'check' : 'fix'; + // Default to 'check', not 'fix': backlinks jobs submitted with an empty + // payload (e.g. the sync→embed→backlinks chains enqueued after ingestion) + // must never rewrite tracked brain pages with generated "Referenced in" + // timeline bullets. Mirrors the documented intent in src/core/cycle.ts + // (runPhaseBacklinks). The filesystem fixer stays available explicitly + // via '{"action":"fix"}' or `gbrain check-backlinks fix`. + const action: 'check' | 'fix' = job.data.action === 'fix' ? 'fix' : 'check'; const dir = typeof job.data.dir === 'string' ? job.data.dir : (await engine.getConfig('sync.repo_path')) ?? '.'; diff --git a/src/commands/lint.ts b/src/commands/lint.ts index e14e05456..54b81c290 100644 --- a/src/commands/lint.ts +++ b/src/commands/lint.ts @@ -127,7 +127,12 @@ export function lintContent(content: string, filePath: string, opts: LintContent } // Rule: Wrapping code fences (```markdown ... ```) - if (content.match(/^```(?:markdown|md)\s*\n/m) && content.match(/\n```\s*$/m)) { + // Detector intentionally has NO /m flag so ^/$ match start/end of the whole + // file, not inner lines. Keeps detector in sync with fixContent() below, + // which also has no /m flag. Without this, lint reports "fixable" false + // positives on any page that simply contains a ```markdown code block, but + // fixContent can never strip them (its regex only matches whole-file wrappers). + if (content.match(/^```(?:markdown|md)\s*\n/) && content.match(/\n```\s*$/)) { issues.push({ file: filePath, line: 1, rule: 'code-fence-wrap', message: 'Page wrapped in ```markdown code fences (LLM artifact)', diff --git a/src/commands/models.ts b/src/commands/models.ts index 8b444370b..b44e8f506 100644 --- a/src/commands/models.ts +++ b/src/commands/models.ts @@ -536,7 +536,20 @@ function shouldSkipProvider(modelStr: string, skip: string[]): boolean { export async function runModels(engine: BrainEngine, args: string[]): Promise { const json = args.includes('--json'); - const sub = args[1] === 'doctor' ? 'doctor' : args[1] === 'help' || args.includes('--help') || args.includes('-h') ? 'help' : 'read'; + // args is `subArgs` from cli.ts `handleCliOnly` — the leading 'models' + // token has already been stripped. The subcommand is at args[0], NOT + // args[1]. Pre-fix this check was `args[1]`, so `gbrain models doctor` + // silently fell through to the read view. The doctor probe path was + // unreachable from the CLI. + // + // --help honored FIRST so `gbrain models doctor --help` shows usage + // instead of running network probes (which would spend tokens or + // exit nonzero when the user only asked for help). Pre-fix the + // args[1] ternary happened to dodge this by always falling through + // to the args.includes('--help') branch; the args[0] rewrite needs + // explicit ordering to preserve that behavior. + const hasHelp = args.includes('--help') || args.includes('-h') || args[0] === 'help'; + const sub = hasHelp ? 'help' : args[0] === 'doctor' ? 'doctor' : 'read'; if (sub === 'help') { process.stdout.write( diff --git a/src/commands/serve-http.ts b/src/commands/serve-http.ts index 2cbae64bc..9b01a183f 100644 --- a/src/commands/serve-http.ts +++ b/src/commands/serve-http.ts @@ -843,6 +843,21 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption // reverse proxies / tunnels; default to localhost for dev. const issuerUrl = new URL(publicUrl || `http://localhost:${port}`); + // MCP authorization spec (2025-06-18 draft §5.1) and RFC 9728 require the + // protected resource server to return its discovery metadata URL in the + // WWW-Authenticate header on 401 responses: + // + // WWW-Authenticate: Bearer resource_metadata="" + // + // Clients (claude.ai, Cursor, every other MCP-aware OAuth client) use that + // URL to find the authorization-server discovery doc + token endpoint + // without the user having to paste those URLs manually. Pre-fix the header + // shipped `Bearer error="invalid_token", ...` with no resource_metadata + // parameter, so MCP clients couldn't begin the OAuth flow from a fresh + // 401 — they would silently fail to connect with a generic "couldn't + // reach the MCP server" error. + const resourceMetadataUrl = `${issuerUrl.toString().replace(/\/$/, '')}/.well-known/oauth-protected-resource`; + // F9: cookie `secure` flag honors both the request's TLS state (req.secure // is set when express trust-proxy lands an X-Forwarded-Proto: https) AND // the operator's declared issuer protocol (so a Cloudflare-tunnel deploy @@ -1601,7 +1616,7 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption res.status(405).json({ jsonrpc: '2.0', error: { code: -32000, message: 'Method not allowed' }, id: null }); }); - app.post('/mcp', requireBearerAuth({ verifier: oauthProvider }), async (req: Request, res: Response) => { + app.post('/mcp', requireBearerAuth({ verifier: oauthProvider, resourceMetadataUrl }), async (req: Request, res: Response) => { const startTime = Date.now(); const authInfo = (req as any).auth as AuthInfo; @@ -1944,7 +1959,7 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption app.post( '/ingest', ingestRateLimiter, - requireBearerAuth({ verifier: oauthProvider, requiredScopes: ['write'] }), + requireBearerAuth({ verifier: oauthProvider, requiredScopes: ['write'], resourceMetadataUrl }), express.raw({ type: '*/*', limit: ingestMaxBytes }), async (req: Request, res: Response) => { const startTime = Date.now(); diff --git a/src/commands/think.ts b/src/commands/think.ts index e3c10882f..9ce9ee099 100644 --- a/src/commands/think.ts +++ b/src/commands/think.ts @@ -6,7 +6,7 @@ * degrades to gather-only output with a warning if missing. */ import type { BrainEngine } from '../core/engine.ts'; -import { runThink, persistSynthesis } from '../core/think/index.ts'; +import { runThink, persistSynthesis, stripGapsSection } from '../core/think/index.ts'; import { loadConfig, isThinClient } from '../core/config.ts'; import { callRemoteTool, unpackToolResult } from '../core/mcp-client.ts'; @@ -157,7 +157,7 @@ prints what would have been the input (exit 0). // Human-readable output console.log(`# ${question}\n`); - console.log(result.answer); + console.log(stripGapsSection(result.answer)); console.log(''); if (result.gaps.length > 0) { console.log('## Gaps'); diff --git a/src/core/ai/dims.ts b/src/core/ai/dims.ts index 7961516a5..b88a4e175 100644 --- a/src/core/ai/dims.ts +++ b/src/core/ai/dims.ts @@ -263,6 +263,15 @@ export function dimsProviderOptions( if (modelId === 'text-embedding-v3' || modelId === 'embedding-3') { return { openaiCompatible: { dimensions: dims } }; } + // Qwen3-Embedding family on Ollama (and any other openai-compatible + // provider serving it) supports Matryoshka truncation via `dimensions`. + // Native sizes: 0.6B=1024, 4B=2560, 8B=4096. Without `dimensions`, + // Ollama returns the native size and brains configured for narrower + // widths hard-fail with a dim-mismatch error. Pattern match the bare + // model name + any `:tag` (e.g. `qwen3-embedding:4b`, `qwen3-embedding:0.6b`). + if (modelId === 'qwen3-embedding' || modelId.startsWith('qwen3-embedding:')) { + return { openaiCompatible: { dimensions: dims } }; + } // MiniMax embo-01 takes a `type: 'db' | 'query'` field for asymmetric // retrieval. Today still hardcoded to 'db' for back-compat — opting // into the new inputType seam is a follow-up (see plan's deferred diff --git a/src/core/ai/gateway.ts b/src/core/ai/gateway.ts index 8a2674484..970d6b606 100644 --- a/src/core/ai/gateway.ts +++ b/src/core/ai/gateway.ts @@ -599,6 +599,8 @@ function warnRecipesMissingBatchTokens(): void { // LiteLLM proxy, llama-server) — they ship without a static cap because // the cap depends on a user-launched server. Warning is noise for them. if (embedding.no_batch_cap === true) continue; + // A declared item-count cap is a real batch cap — no warning needed. + if (embedding.max_batch_items !== undefined) continue; if (_warnedRecipes.has(recipe.id)) continue; _warnedRecipes.add(recipe.id); // eslint-disable-next-line no-console @@ -1517,10 +1519,17 @@ export async function embed(texts: string[], opts?: EmbedOpts): Promise capBatchItems(b, maxBatchItems)) + : tokenBatches; + const allEmbeddings: Float32Array[] = []; let _embedThrew = false; try { @@ -1596,6 +1605,23 @@ export function splitByTokenBudget( return batches; } +/** + * Split a batch into sub-batches of at most `maxItems` inputs. Enforces a + * hard COUNT cap that the token-budget split can't (many tiny inputs fit + * under any token budget). Used for endpoints like llama.cpp's llama-server + * that reject requests exceeding their launch batch size. + * + * @internal exported for tests; not part of the public gateway API. + */ +export function capBatchItems(texts: string[], maxItems: number): string[][] { + if (maxItems <= 0 || texts.length <= maxItems) return [texts]; + const batches: string[][] = []; + for (let i = 0; i < texts.length; i += maxItems) { + batches.push(texts.slice(i, i + maxItems)); + } + return batches; +} + /** * Returns true if the error looks like a provider batch-token-limit error. * diff --git a/src/core/ai/recipes/llama-server.ts b/src/core/ai/recipes/llama-server.ts index 212bd5478..a51650b27 100644 --- a/src/core/ai/recipes/llama-server.ts +++ b/src/core/ai/recipes/llama-server.ts @@ -35,9 +35,12 @@ export const llamaServer: Recipe = { trust_custom_dims: true, // #2271: user knows the launched model's native dim cost_per_1m_tokens_usd: 0, price_last_verified: '2026-05-10', - // llama-server's batch capacity is set by `--ctx-size` at launch - // time; no static cap to declare. v0.32 (#779). - no_batch_cap: true, + // llama-server enforces a hard request-COUNT cap equal to its launch + // batch size (`--batch-size`, default 32): it rejects requests with + // more inputs with `batch size N > maximum allowed batch size 32`. + // The token-budget split can't bound item count, so cap it here. A + // server launched with a larger `-b` can raise this. v0.32 (#779). + max_batch_items: 32, }, }, /** diff --git a/src/core/ai/recipes/openrouter.ts b/src/core/ai/recipes/openrouter.ts index 055848ac8..86782cd59 100644 --- a/src/core/ai/recipes/openrouter.ts +++ b/src/core/ai/recipes/openrouter.ts @@ -23,6 +23,14 @@ import type { Recipe } from '../types.ts'; * envelope, not every individual model's capability. When in doubt about a * specific model, check https://openrouter.ai/models. * + * Reranker: `/api/v1/rerank` proxies cross-encoder rerankers (Cohere v3.5/4-fast/4-pro + * and NVIDIA Nemotron VL). Wire shape matches `gateway.rerank()`: + * `{ query, documents, model }` → `{ results: [{ index, relevance_score }] }`. + * Unlike embedding/chat, the reranker path strictly enforces the `models` + * allowlist (no openai-compat bypass) — adding new rerank models requires a + * recipe edit. Cohere bills per-search; the `cost_per_1m_tokens_usd` value + * is a pseudo-rate for the budget tracker's `chars/4` heuristic. + * * Attribution: OpenRouter recommends `HTTP-Referer` (required for app * attribution) + `X-OpenRouter-Title` (preferred; `X-Title` kept as * back-compat alias per OR docs). Defaults to `https://gbrain.ai` / `gbrain`; @@ -99,6 +107,30 @@ export const openrouter: Recipe = { // Let upstream errors surface per-model. price_last_verified: '2026-05-20', }, + reranker: { + models: [ + 'cohere/rerank-v3.5', + 'cohere/rerank-4-fast', + 'cohere/rerank-4-pro', + 'nvidia/llama-nemotron-rerank-vl-1b-v2:free', + ], + default_model: 'cohere/rerank-v3.5', + // Cohere bills per-search, not per-token. This is a pseudo-per-1M rate + // for the budget tracker's heuristic (estimates tokens as chars/4). + // At ~4K chars/search the tracker estimates ~$0.00025 — in the right + // ballpark for the per-search bill. Patch budget-tracker.ts to honour a + // `cost_per_search_usd` field for exact accounting. + cost_per_1m_tokens_usd: 0.001, + price_last_verified: '2026-06-13', + // OpenRouter doesn't publish an explicit payload cap; 5MB matches + // ZeroEntropy's upstream limit and the gateway's pre-flight ceiling. + max_payload_bytes: 5_000_000, + // OR serves /rerank under /api/v1. base_url_default already ends in /v1, + // so gateway concatenates to …/api/v1/rerank. + path: '/rerank', + // OpenRouter rerank is fast (<200 ms p50); 5 s covers cold path safely. + default_timeout_ms: 5_000, + }, }, setup_hint: 'Get an API key at https://openrouter.ai/settings/keys, then `export OPENROUTER_API_KEY=...` and use `openrouter:/`. Optional overrides: OPENROUTER_BASE_URL (proxy), OPENROUTER_REFERER (attribution URL), OPENROUTER_TITLE (attribution name).', diff --git a/src/core/ai/types.ts b/src/core/ai/types.ts index 798551eec..8fc785e3d 100644 --- a/src/core/ai/types.ts +++ b/src/core/ai/types.ts @@ -54,6 +54,16 @@ export interface EmbeddingTouchpoint { * `max_batch_tokens` is also set. */ safety_factor?: number; + /** + * Maximum number of inputs per embedding request. Some endpoints enforce a + * hard COUNT cap independent of token budget — notably llama.cpp's + * `llama-server`, which rejects requests with more inputs than its launch + * batch size (e.g. `batch size 100 > maximum allowed batch size 32`). The + * token-budget pre-split cannot bound item count (many tiny chunks fit under + * any token budget), so this is enforced as a separate hard re-split after + * the token split. When unset, no count cap is applied. + */ + max_batch_items?: number; /** * v0.27.1: when true, at least one model in this recipe accepts image * inputs via a multimodal embedding endpoint (e.g. Voyage's diff --git a/src/core/check-resolvable.ts b/src/core/check-resolvable.ts index a82b4a2c1..9f2f72980 100644 --- a/src/core/check-resolvable.ts +++ b/src/core/check-resolvable.ts @@ -217,9 +217,19 @@ export function parseResolverEntries(resolverContent: string): ResolverEntry[] { // `skillsDir/*/SKILL.md` when manifest.json is missing — the scenario // needed for AGENTS.md-only OpenClaw deployments. See D-CX-12 / F-ENG-1. -/** Simple YAML frontmatter parser — extracts triggers array if present. */ -function extractTriggers(skillContent: string): string[] { - const fmMatch = skillContent.match(/^---\n([\s\S]*?)\n---/); +/** + * Simple YAML frontmatter parser — extracts triggers array if present. + * + * Normalizes CRLF → LF before parsing so Windows checkouts (where + * `core.autocrlf=true` is the default) parse correctly. Without this, + * the `^---\n` and `^triggers:\s*\n` regexes never match because the + * file content is `---\r\n` / `triggers:\r\n`, and every skill on + * Windows is reported as `mece_gap` regardless of its actual content. + * CI runs on Ubuntu-only so the bug only surfaces in user environments. + */ +export function extractTriggers(skillContent: string): string[] { + const content = skillContent.replace(/\r\n/g, '\n'); + const fmMatch = content.match(/^---\n([\s\S]*?)\n---/); if (!fmMatch) return []; const fm = fmMatch[1]; const triggersMatch = fm.match(/^triggers:\s*\n((?:\s+-\s+.+\n?)*)/m); diff --git a/src/core/chunkers/code.ts b/src/core/chunkers/code.ts index 1ff1d2efd..5578a290a 100644 --- a/src/core/chunkers/code.ts +++ b/src/core/chunkers/code.ts @@ -168,6 +168,15 @@ export interface CodeChunkOptions { largeChunkThresholdTokens?: number; fallbackChunkSizeWords?: number; fallbackOverlapWords?: number; + /** + * Hard upper bound (estimated tokens) on any single emitted chunk. A node + * the AST splitter can't break up (a giant object/array literal, a single + * huge assignment, a massive template literal) would otherwise be emitted + * whole and rejected by the embedder ("input exceeds context length"). + * Chunks over this budget are recursively re-split. Default 2000 fits the + * smallest common embedder context (e.g. nomic-embed-text, 2048). + */ + maxChunkTokens?: number; } /** @@ -549,6 +558,7 @@ export function parseWithTimeout( } const DEFAULT_CHUNKER_TIMEOUT_MS = 30_000; +const DEFAULT_MAX_CHUNK_TOKENS = 2000; function resolveChunkerTimeoutMs(): number { const raw = process.env.GBRAIN_CHUNKER_TIMEOUT_MS; @@ -706,9 +716,9 @@ export async function chunkCodeTextFull( } if (chunks.length === 0) { - return { chunks: fallbackChunks(source, filePath, language, opts), edges: rawEdges }; + return { chunks: capOversizedChunks(fallbackChunks(source, filePath, language, opts), filePath, language, opts), edges: rawEdges }; } - return { chunks: mergeSmallSiblings(chunks, chunkTarget), edges: rawEdges }; + return { chunks: capOversizedChunks(mergeSmallSiblings(chunks, chunkTarget), filePath, language, opts), edges: rawEdges }; } catch { return { chunks: fallbackChunks(source, filePath, language, opts), edges: [] }; } finally { @@ -814,6 +824,73 @@ function buildMergedChunk(group: CodeChunk[], index: number): CodeChunk { }; } +/** + * Final safety net: guarantee no emitted chunk exceeds the embedder's context + * budget. tree-sitter splitting (splitLargeNode) can only break up a node that + * exposes a `body` with >= 2 named children. A node without one — a giant + * object/array literal, a single huge assignment, a massive template literal — + * is emitted whole, producing a chunk far larger than the embedder accepts. + * The embedder then rejects it ("input exceeds context length") and the chunk + * is never embedded. Recursively re-split any over-budget chunk; fall back to a + * hard character split for pathological no-whitespace content (e.g. a minified + * one-liner) where word/line splitting can't get under budget. + */ +function capOversizedChunks( + chunks: CodeChunk[], + filePath: string, + language: SupportedCodeLanguage, + opts: CodeChunkOptions, +): CodeChunk[] { + const cap = opts.maxChunkTokens ?? DEFAULT_MAX_CHUNK_TOKENS; + if (!chunks.some((c) => estimateTokens(c.text) > cap)) return chunks; + const out: CodeChunk[] = []; + for (const c of chunks) { + if (estimateTokens(c.text) <= cap) { + out.push({ ...c, index: out.length }); + continue; + } + // Strip the structured header ("[Lang] path:N-M symbol\n\n") so the splitter + // works on the raw body; buildChunk re-adds a header to each piece. + const body = c.text.replace(/^\[[^\]]+\] [^\n]+\n\n/, ''); + for (const piece of splitToTokenBudget(body, cap, opts)) { + if (!piece.trim()) continue; + out.push(buildChunk({ + body: piece, + filePath, + language, + symbolName: c.metadata.symbolName, + symbolType: c.metadata.symbolType, + startLine: c.metadata.startLine, + endLine: c.metadata.endLine, + index: out.length, + parentSymbolPath: c.metadata.parentSymbolPath, + })); + } + } + return out; +} + +/** Split `text` into pieces each estimated <= cap tokens. Word/line-aware + * (recursiveChunk) first; a hard character split is the last resort for + * content with no whitespace to break on. */ +function splitToTokenBudget(text: string, cap: number, opts: CodeChunkOptions): string[] { + const out: string[] = []; + const pieces = recursiveChunk(text, { + chunkSize: opts.fallbackChunkSizeWords ?? 300, + chunkOverlap: opts.fallbackOverlapWords ?? 50, + }).map((p) => p.text); + for (const piece of pieces) { + if (estimateTokens(piece) <= cap) { + out.push(piece); + continue; + } + // ~3.5 chars/token is a conservative cl100k estimate for source text. + const charBudget = Math.max(1, Math.floor(cap * 3.5)); + for (let i = 0; i < piece.length; i += charBudget) out.push(piece.slice(i, i + charBudget)); + } + return out; +} + // ---------- Internals ---------- function fallbackChunks( diff --git a/src/core/config.ts b/src/core/config.ts index 4ca00cdc9..e81954886 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -620,7 +620,10 @@ export function loadConfig(): GBrainConfig | null { * size the schema and must be stable across engine connect. */ export async function loadConfigWithEngine( - engine: { getConfig(key: string): Promise }, + engine: { + getConfig(key: string): Promise; + listConfigKeys?(prefix: string): Promise; + }, base?: GBrainConfig | null, ): Promise { // Codex /ship finding #3: when there's no file config AND no env DB URL, @@ -657,11 +660,31 @@ export async function loadConfigWithEngine( return undefined; } } + async function dbPrefixMap(prefix: string): Promise | undefined> { + if (typeof engine.listConfigKeys !== 'function') return undefined; + let keys: string[]; + try { + keys = await engine.listConfigKeys(prefix); + } catch { + return undefined; + } + + const out: Record = {}; + for (const key of keys.sort()) { + if (!key.startsWith(prefix)) continue; + const leaf = key.slice(prefix.length); + if (!leaf) continue; + const value = await dbStr(key); + if (value !== undefined) out[leaf] = value; + } + return Object.keys(out).length > 0 ? out : undefined; + } const dbMultimodal = await dbBool('embedding_multimodal'); const dbMultimodalModel = await dbStr('embedding_multimodal_model'); const dbOcr = await dbBool('embedding_image_ocr'); const dbOcrModel = await dbStr('embedding_image_ocr_model'); + const dbProviderBaseUrls = await dbPrefixMap('provider_base_urls.'); // v0.36 (D7) — embedding-column registry merge. Stored as JSON string in // the config table. Parse + shape-check here; full registry validation // (regex on keys, type/dim/provider field shapes) runs in the resolver at @@ -685,6 +708,15 @@ export async function loadConfigWithEngine( if (merged.embedding_image_ocr_model === undefined && dbOcrModel !== undefined) { merged.embedding_image_ocr_model = dbOcrModel; } + if (dbProviderBaseUrls !== undefined) { + const next = { ...(merged.provider_base_urls ?? {}) }; + for (const [providerId, baseUrl] of Object.entries(dbProviderBaseUrls)) { + if (next[providerId] === undefined) next[providerId] = baseUrl; + } + if (Object.keys(next).length > 0) { + merged.provider_base_urls = next; + } + } if (merged.embedding_columns === undefined && dbEmbeddingColumns !== undefined) { try { const parsed = JSON.parse(dbEmbeddingColumns); diff --git a/src/core/cycle/extract-atoms-drain.ts b/src/core/cycle/extract-atoms-drain.ts index 98a4bfa69..91f364d87 100644 --- a/src/core/cycle/extract-atoms-drain.ts +++ b/src/core/cycle/extract-atoms-drain.ts @@ -90,7 +90,14 @@ export async function runExtractAtomsDrain( // Stop if a batch made zero forward progress — extraction is failing or // everything left is ineligible (e.g. all skipped). Prevents a hot loop // that spends budget without draining. - if (r.extracted === 0 && r.skipped === 0) { stopped = 'no_progress'; break; } + // + // #2144: a zero-ATOM batch can still be progress — tombstoned + // zero-yield pages shrink the backlog without producing atoms. Only + // stop when the backlog count genuinely didn't move. + if (r.extracted === 0 && r.skipped === 0) { + const after = await deps.countRemaining(); + if (after === null || before === null || after >= before) { stopped = 'no_progress'; break; } + } } const remaining = await deps.countRemaining(); diff --git a/src/core/cycle/extract-atoms.ts b/src/core/cycle/extract-atoms.ts index 4c73ea400..82ddf7aa1 100644 --- a/src/core/cycle/extract-atoms.ts +++ b/src/core/cycle/extract-atoms.ts @@ -163,10 +163,20 @@ interface ExtractedAtom { body: string; source_quote?: string; lesson?: string; + /** + * 1-3 kebab-case topic labels for concept clustering. Consumed by + * synthesize_concepts (groups atoms by `frontmatter.concepts`; only + * labels shared by >=2 atoms materialize a concept page, so the prompt + * biases reuse-over-coinage). #2123. + */ + concepts?: string[]; virality_score?: number; emotional_register?: string; } +/** kebab-case validator for concept labels ("captive-portal", "channel-pricing"). */ +const CONCEPT_LABEL_RE = /^[a-z0-9]+(-[a-z0-9]+)*$/; + const EXTRACT_PROMPT = `You extract atomic content nuggets from a transcript. An atom is a single-source, self-contained idea that could become a tweet, @@ -177,12 +187,17 @@ quote, or short essay angle. Each atom must: Output a JSON array of atoms (1-3 per transcript, never more than 3). Each atom: {title (≤80 chars), atom_type, body (2-4 sentences), -source_quote (verbatim ≤200 chars), lesson (one sentence), virality_score -(0-100), emotional_register (one of: shocking, inspiring, funny, sobering, -practical, controversial)}. +source_quote (verbatim ≤200 chars), lesson (one sentence), concepts +(1-3 topic labels), virality_score (0-100), emotional_register (one of: +shocking, inspiring, funny, sobering, practical, controversial)}. atom_type MUST be one of: ${ATOM_TYPES.join(', ')}. +concepts are kebab-case English TOPIC labels used to cluster atoms into +concept pages (e.g. "captive-portal", "channel-pricing-strategy") — never +entity or brand names. Use the same label for the same topic across atoms; +prefer a label you already used over coining a near-synonym. + Output ONLY the JSON array, no prose.`; interface DiscoveredPage { @@ -226,6 +241,7 @@ export async function discoverExtractablePages( AND COALESCE(p.frontmatter->>'imported_from', '') <> 'markdown-greenfield' AND COALESCE(p.frontmatter->>'dream_generated', '') <> 'true' AND length(COALESCE(p.compiled_truth, '')) >= $3 + AND COALESCE(p.frontmatter->>'atoms_scan_hash', '') <> substring(p.content_hash from 1 for 16) ${hasFilter ? "AND p.slug = ANY($5::text[])" : ''} AND NOT EXISTS ( SELECT 1 @@ -298,6 +314,7 @@ export async function countExtractAtomsBacklog( AND COALESCE(p.frontmatter->>'imported_from', '') <> 'markdown-greenfield' AND COALESCE(p.frontmatter->>'dream_generated', '') <> 'true' AND length(COALESCE(p.compiled_truth, '')) >= $3 + AND COALESCE(p.frontmatter->>'atoms_scan_hash', '') <> substring(p.content_hash from 1 for 16) AND NOT EXISTS ( SELECT 1 FROM pages atom WHERE atom.type = 'atom' AND atom.source_id = $1 @@ -311,6 +328,7 @@ export async function countExtractAtomsBacklog( AND COALESCE(p.frontmatter->>'imported_from', '') <> 'markdown-greenfield' AND COALESCE(p.frontmatter->>'dream_generated', '') <> 'true' AND length(COALESCE(p.compiled_truth, '')) >= $2 + AND COALESCE(p.frontmatter->>'atoms_scan_hash', '') <> substring(p.content_hash from 1 for 16) AND NOT EXISTS ( SELECT 1 FROM pages atom WHERE atom.type = 'atom' AND atom.source_id = p.source_id @@ -556,6 +574,25 @@ export async function runPhaseExtractAtoms( const atoms = parseAtomsResponse(result.text); if (atoms.length === 0) { + // #2144: tombstone zero-yield pages so they stop being rediscovered. + // Idempotency is keyed on atom rows — a page that yields no atoms + // leaves no row, so pre-fix it re-entered the discovery window every + // run (wedging --drain with a false no_progress and re-spending + // nightly budget on the same pages). Stamp the content hash we + // scanned; discovery skips the page only while its content is + // unchanged (edits re-eligibilize, mirroring atom-row staleness). + // Only stamped after a SUCCESSFUL chat call — LLM failures take the + // catch path below and stay retryable. + if (!opts.dryRun && item.kind === 'page') { + try { + await engine.executeRaw( + `UPDATE pages + SET frontmatter = frontmatter || jsonb_build_object('atoms_scan_hash', $1::text) + WHERE source_id = $2 AND slug = $3 AND deleted_at IS NULL`, + [item.contentHash.slice(0, 16), sourceId, item.slug], + ); + } catch { /* fail-soft: page stays rediscoverable */ } + } if (item.kind === 'transcript') transcriptsProcessed++; else pagesProcessed++; continue; @@ -585,6 +622,7 @@ export async function runPhaseExtractAtoms( source_hash: item.contentHash.slice(0, 16), ...(atom.source_quote && { source_quote: atom.source_quote }), ...(atom.lesson && { lesson: atom.lesson }), + ...(atom.concepts && atom.concepts.length > 0 && { concepts: atom.concepts }), ...(atom.virality_score !== undefined && { virality_score: atom.virality_score }), ...(atom.emotional_register && { emotional_register: atom.emotional_register }), extracted_at: new Date().toISOString(), @@ -721,6 +759,13 @@ export function parseAtomsResponse(raw: string): ExtractedAtom[] { body, source_quote: typeof obj.source_quote === 'string' ? obj.source_quote.slice(0, 500) : undefined, lesson: typeof obj.lesson === 'string' ? obj.lesson : undefined, + concepts: (() => { + if (!Array.isArray(obj.concepts)) return undefined; + const labels = obj.concepts + .filter((c): c is string => typeof c === 'string' && CONCEPT_LABEL_RE.test(c)) + .slice(0, 3); + return labels.length > 0 ? labels : undefined; + })(), virality_score: typeof obj.virality_score === 'number' && obj.virality_score >= 0 && diff --git a/src/core/engine.ts b/src/core/engine.ts index a389d3091..ef04165b4 100644 --- a/src/core/engine.ts +++ b/src/core/engine.ts @@ -1218,11 +1218,21 @@ export interface BrainEngine { * * Uses the `%` trigram operator (GIN-indexed) + the standard `similarity()` * function. Both engines support pg_trgm (PGLite 0.3+, Postgres always). + * + * `sourceId` constrains the search to a single source and filters out + * soft-deleted pages. Mirrors the same filters `tryFuzzyMatch` in + * `src/core/entities/resolve.ts` got via #1436 (v0.41.13.0). Omit for the + * historical unscoped behavior — live-mode callers that already know + * the source should pass it to avoid cross-source slug suggestions that + * get silently dropped at the FK filter downstream. Batch-mode callers + * (e.g. `gbrain extract`) intentionally omit it to build a cross-source + * resolution map. */ findByTitleFuzzy( name: string, dirPrefix?: string, minSimilarity?: number, + sourceId?: string, ): Promise<{ slug: string; similarity: number } | null>; /** * v0.34.1 (#861 — P0 leak seal): `opts.sourceId` / `opts.sourceIds` diff --git a/src/core/link-extraction.ts b/src/core/link-extraction.ts index 8f27903e6..8aa203480 100644 --- a/src/core/link-extraction.ts +++ b/src/core/link-extraction.ts @@ -79,11 +79,11 @@ export type LinkResolutionType = 'qualified' | 'unqualified'; /** * Directory prefix whitelist. These are the top-level slug dirs the extractor * recognizes as entity references. Upstream canonical + our extensions: - * - Gbrain canonical: people, companies, meetings, concepts, deal, civic, project, source, media, yc, projects + * - Gbrain canonical: people, companies, meetings, concepts, deal, civic, project, source, media, yc, projects, reference * - Our domain extensions: tech, finance, personal, openclaw (domain-organized wikis) * - Our entity prefix: entities (we kept some legacy entities/projects/ pages) */ -const DIR_PATTERN = '(?:people|companies|meetings|concepts|deal|civic|project|projects|source|media|yc|tech|finance|personal|openclaw|entities)'; +const DIR_PATTERN = '(?:people|companies|meetings|concepts|deal|civic|project|projects|source|media|yc|tech|finance|personal|openclaw|entities|reference)'; /** * Match `[Name](path)` markdown links pointing to entity directories. @@ -980,10 +980,14 @@ export function makeResolver( // Step 3: pg_trgm fuzzy title match — both modes. Tries each hint in // order; first hint with a ≥0.55 similarity match wins. If no hints, - // try the whole pages table. + // try the whole pages table. When opts.sourceId is set, the fuzzy + // search is constrained to that source (and skips soft-deleted pages) + // so cross-source slug suggestions don't get silently dropped at the + // FK filter downstream. Mirrors the same scope fix `tryFuzzyMatch` got + // via #1436. const searchHints = hints.length > 0 ? hints : [undefined]; for (const hint of searchHints) { - const match = await engine.findByTitleFuzzy(trimmed, hint, 0.55); + const match = await engine.findByTitleFuzzy(trimmed, hint, 0.55, opts.sourceId); if (match) { cache.set(cacheKey, match.slug); return match.slug; diff --git a/src/core/multi-source-drift.ts b/src/core/multi-source-drift.ts index 2064c48f4..e01f34ec4 100644 --- a/src/core/multi-source-drift.ts +++ b/src/core/multi-source-drift.ts @@ -87,6 +87,11 @@ function walkMarkdownAndMdxFiles( for (const entry of entries) { if (truncated) return; if (entry.startsWith('.')) continue; + // Skip heavy non-content dirs so the walk doesn't exhaust the time + // budget on dependency/build trees (node_modules can be 50k+ files + // with zero .md). These are never gbrain page sources. + if (entry === 'node_modules' || entry === 'dist' || entry === 'build' || + entry === '.next' || entry === 'vendor' || entry === 'target') continue; const full = join(d, entry); let isDir = false; try { @@ -95,6 +100,9 @@ function walkMarkdownAndMdxFiles( continue; } if (isDir) { + // Time check on directory descent too, so a deep dependency-free + // tree still respects the deadline even before any .md is found. + if (Date.now() >= deadlineMs) { truncated = true; return; } walk(full); continue; } diff --git a/src/core/operations.ts b/src/core/operations.ts index ab2d4cbf1..68f5a56e1 100644 --- a/src/core/operations.ts +++ b/src/core/operations.ts @@ -1153,7 +1153,11 @@ async function runAutoLink( // Live-mode resolver: per-put throwaway cache, pg_trgm + optional search. // Issue #972 (codex [P1]): pass sourceId so basename resolution stays - // within this page's source — no cross-source basename edges. + // within this page's source — no cross-source basename edges. Also scopes + // the fuzzy fallback (findByTitleFuzzy) to the same source the put_page is + // targeting — without it, cross-source slug suggestions get silently dropped + // at the FK filter and the link looks like it failed to resolve. Twin of + // #1436's `tryFuzzyMatch` fix. const resolver = makeResolver(engine, { mode: 'live', sourceId: opts?.sourceId }); // Issue #972: opt-in bare-wikilink basename resolution. Off by default. const globalBasename = await isGlobalBasenameEnabled(engine); diff --git a/src/core/pglite-engine.ts b/src/core/pglite-engine.ts index 257090a23..f74e4e964 100644 --- a/src/core/pglite-engine.ts +++ b/src/core/pglite-engine.ts @@ -1060,6 +1060,16 @@ export class PGLiteEngine implements BrainEngine { RETURNING id, source_id, slug, type, title, compiled_truth, timeline, frontmatter, content_hash, created_at, updated_at, effective_date, effective_date_source, import_filename, source_kind, source_uri, ingested_via, ingested_at`, [sourceId, slug, page.type, pageKind, page.title, page.compiled_truth, page.timeline || '', JSON.stringify(frontmatter), hash, effectiveDate, effectiveDateSource, importFilename, chunkerVersion, sourcePath, sourceKind, sourceUri, ingestedVia, ingestedAt] ); + // PGLite can return zero rows from INSERT ... ON CONFLICT DO UPDATE ... + // RETURNING in no-op/trigger edge cases, which made rowToPage(undefined) + // throw "undefined is not an object (evaluating 'row.deleted_at')" and + // skip the file during sync. The row WAS written, so re-read instead of + // crashing. + if (rows.length === 0) { + const reread = await this.getPage(slug, { sourceId }); + if (reread) return reread; + throw new Error(`putPage: RETURNING produced no row for ${sourceId}/${slug}`); + } return rowToPage(rows[0] as Record); } @@ -2912,22 +2922,41 @@ export class PGLiteEngine implements BrainEngine { name: string, dirPrefix?: string, minSimilarity: number = 0.55, + sourceId?: string, ): Promise<{ slug: string; similarity: number } | null> { // Inline threshold comparison instead of `SET LOCAL pg_trgm.similarity_threshold`. // The GUC only scopes to the current transaction and pglite auto-commits each // .query() call, so the SET LOCAL would be a no-op. Using similarity() >= $N // directly gives predictable behavior. Tie-breaker: sort by slug so re-runs // pick the same winner. + // + // `sourceId` + `deleted_at IS NULL` mirror the filters `tryFuzzyMatch` in + // `src/core/entities/resolve.ts` got via #1436 (v0.41.13.0). Without them, + // fuzzy resolution could suggest cross-source slugs that the caller then + // silently drops at the FK filter — making it look like the match failed + // when in fact it picked the wrong page. const prefixPattern = dirPrefix ? `${dirPrefix}/%` : '%'; - const { rows } = await this.db.query( - `SELECT slug, similarity(title, $1) AS sim - FROM pages - WHERE similarity(title, $1) >= $3 - AND slug LIKE $2 - ORDER BY sim DESC, slug ASC - LIMIT 1`, - [name, prefixPattern, minSimilarity] - ); + const { rows } = sourceId + ? await this.db.query( + `SELECT slug, similarity(title, $1) AS sim + FROM pages + WHERE similarity(title, $1) >= $3 + AND slug LIKE $2 + AND source_id = $4 + AND deleted_at IS NULL + ORDER BY sim DESC, slug ASC + LIMIT 1`, + [name, prefixPattern, minSimilarity, sourceId] + ) + : await this.db.query( + `SELECT slug, similarity(title, $1) AS sim + FROM pages + WHERE similarity(title, $1) >= $3 + AND slug LIKE $2 + ORDER BY sim DESC, slug ASC + LIMIT 1`, + [name, prefixPattern, minSimilarity] + ); if (rows.length === 0) return null; const row = rows[0] as { slug: string; sim: number }; return { slug: row.slug, similarity: row.sim }; @@ -5837,6 +5866,11 @@ export class PGLiteEngine implements BrainEngine { params.push(escaped); prefixCondition = `AND p.slug LIKE $${params.length} ESCAPE '\\'`; } + // TIM-37: exclude briefing pages from their own Brain Pulse. See the + // matching block in postgres-engine.ts getRecentSalience() for context. + const excludeBriefings = !(slugPrefix && slugPrefix.startsWith('briefings')) + ? `AND p.slug NOT LIKE 'briefings/%'` + : ''; params.push(limit); const limitParam = `$${params.length}`; @@ -5872,6 +5906,7 @@ export class PGLiteEngine implements BrainEngine { LEFT JOIN takes t ON t.page_id = p.id AND t.active = TRUE WHERE GREATEST(p.updated_at, COALESCE(p.salience_touched_at, p.updated_at)) >= $1::timestamptz ${prefixCondition} + ${excludeBriefings} GROUP BY p.id ORDER BY score DESC LIMIT ${limitParam}`, diff --git a/src/core/postgres-engine.ts b/src/core/postgres-engine.ts index 0e8f658f8..6deaad784 100644 --- a/src/core/postgres-engine.ts +++ b/src/core/postgres-engine.ts @@ -3082,6 +3082,7 @@ export class PostgresEngine implements BrainEngine { name: string, dirPrefix?: string, minSimilarity: number = 0.55, + sourceId?: string, ): Promise<{ slug: string; similarity: number } | null> { const sql = this.sql; // Use the `similarity()` function directly with an explicit threshold @@ -3094,15 +3095,33 @@ export class PostgresEngine implements BrainEngine { // Tie-breaker: sort by slug after similarity so re-runs return the // same winner when multiple pages score equally (prevents churn // in put_page auto-link reconciliation). + // + // `sourceId` + `deleted_at IS NULL` mirror the filters `tryFuzzyMatch` + // in `src/core/entities/resolve.ts` got via #1436 (v0.41.13.0). Without + // them, fuzzy resolution could suggest cross-source slugs that the + // caller then silently drops at the FK filter in + // `operations.ts:reconcileLinks` (the `allSlugs` filter) — making it + // look like the match failed when in fact it picked the wrong page. const prefixPattern = dirPrefix ? `${dirPrefix}/%` : '%'; - const rows = await sql` - SELECT slug, similarity(title, ${name}) AS sim - FROM pages - WHERE similarity(title, ${name}) >= ${minSimilarity} - AND slug LIKE ${prefixPattern} - ORDER BY sim DESC, slug ASC - LIMIT 1 - `; + const rows = sourceId + ? await sql` + SELECT slug, similarity(title, ${name}) AS sim + FROM pages + WHERE similarity(title, ${name}) >= ${minSimilarity} + AND slug LIKE ${prefixPattern} + AND source_id = ${sourceId} + AND deleted_at IS NULL + ORDER BY sim DESC, slug ASC + LIMIT 1 + ` + : await sql` + SELECT slug, similarity(title, ${name}) AS sim + FROM pages + WHERE similarity(title, ${name}) >= ${minSimilarity} + AND slug LIKE ${prefixPattern} + ORDER BY sim DESC, slug ASC + LIMIT 1 + `; if (rows.length === 0) return null; const row = rows[0] as { slug: string; sim: number }; return { slug: row.slug, similarity: row.sim }; @@ -6153,6 +6172,13 @@ export class PostgresEngine implements BrainEngine { const prefixCondition = slugPrefix ? sql`AND p.slug LIKE ${slugPrefix.replace(/[\\%_]/g, (c) => '\\' + c) + '%'} ESCAPE '\\'` : sql``; + // TIM-37: exclude briefing pages from their own Brain Pulse. The cron + // briefing writes to 90_Briefings/, gets re-ingested, and would otherwise + // top tomorrow's salience as pure self-reference. Suppress unless the + // caller explicitly asked for the briefings/ prefix. + const excludeBriefings = !(slugPrefix && slugPrefix.startsWith('briefings')) + ? sql`AND p.slug NOT LIKE 'briefings/%'` + : sql``; // v0.29.1: third score term via buildRecencyComponentSql. Default // 'flat' = v0.29.0 behavior (1 / (1 + days_old)). 'on' opts into the // per-prefix decay map (concepts/ evergreen, daily/ aggressive, etc.). @@ -6186,6 +6212,7 @@ export class PostgresEngine implements BrainEngine { LEFT JOIN takes t ON t.page_id = p.id AND t.active = TRUE WHERE GREATEST(p.updated_at, COALESCE(p.salience_touched_at, p.updated_at)) >= ${boundaryIso}::timestamptz ${prefixCondition} + ${excludeBriefings} GROUP BY p.id ORDER BY score DESC LIMIT ${limit} diff --git a/src/core/think/index.ts b/src/core/think/index.ts index 192979d30..8a0918b5f 100644 --- a/src/core/think/index.ts +++ b/src/core/think/index.ts @@ -556,6 +556,40 @@ export async function runThink( }; } +/** + * Strip a "## Gaps" section from an answer body. + * + * `think` returns gaps in the structured `gaps` array, which the CLI and the + * persisted synthesis page render exactly once. The system prompt also used to + * ask for a "Gaps" section inside the answer prose, so a model that still emits + * one would make the output show "## Gaps" twice — once from the prose, once + * from the structured array. This removes the prose section so the structured + * array stays the single source of truth. + * + * Matches a heading line `## Gaps` (level 2-6, case-insensitive) and removes it + * through the next heading of the same-or-higher level, or end of string. + * Returns the input unchanged when there is no such section. + */ +export function stripGapsSection(answer: string): string { + if (!answer) return answer; + const lines = answer.split('\n'); + let start = -1; + let level = 0; + for (let i = 0; i < lines.length; i++) { + const m = /^(#{2,6})\s+gaps\s*$/i.exec(lines[i]); + if (m) { start = i; level = m[1].length; break; } + } + if (start === -1) return answer; + let end = lines.length; + for (let i = start + 1; i < lines.length; i++) { + const h = /^(#{1,6})\s+\S/.exec(lines[i]); + if (h && h[1].length <= level) { end = i; break; } + } + const kept = [...lines.slice(0, start), ...lines.slice(end)].join('\n'); + // Drop trailing blank lines left by removing a trailing section. + return kept.replace(/\s+$/, ''); +} + /** * Persist a synthesis page + its evidence. Returns the saved slug. * Synthesis pages are written under `synthesis/-.md`. @@ -585,7 +619,7 @@ export async function persistSynthesis( const body = [ `# ${result.question}`, '', - result.answer, + stripGapsSection(result.answer), '', result.gaps.length > 0 ? '## Gaps\n\n' + result.gaps.map(g => `- ${g}`).join('\n') : '', ].filter(Boolean).join('\n'); diff --git a/src/core/think/prompt.ts b/src/core/think/prompt.ts index 7107ef729..06a16f84a 100644 --- a/src/core/think/prompt.ts +++ b/src/core/think/prompt.ts @@ -52,19 +52,19 @@ Hard rules: rather than asserting it as established. Confidence is part of the data. - If two takes contradict (different holders, opposite claims), surface BOTH in a "Conflicts" section. Never silently pick one. -- If you cannot answer because the brain doesn't contain the relevant data, say so in the - "Gaps" section. List the specific missing pieces. Do not make up answers. +- If the brain doesn't contain data needed to answer, do NOT make it up. Record each + missing piece in the structured "gaps" array (below), not as a section in the answer prose. - Never instruct the user (no "you should" / "I recommend X"). The brain reports; the user decides. - Output MUST be valid JSON matching the schema below. No prose outside JSON. Output schema: { - "answer": "", + "answer": "", "citations": [ {"page_slug": "people/alice-example", "row_num": 3, "citation_index": 1}, {"page_slug": "companies/acme-example", "row_num": null, "citation_index": 2} ], - "gaps": ["specific missing data point 1", "specific missing data point 2"] + "gaps": ["a specific, self-contained missing-or-stale data point, citing the [slug] where relevant", "another specific gap"] } The "row_num" field is required for take citations and MUST be null for page-only citations.`; @@ -83,7 +83,7 @@ export function buildThinkSystemPrompt(opts: ThinkSystemPromptOpts = {}): string lines.push(`\nThis is a temporal question. Order key claims chronologically when it helps the reader.`); } if (opts.willSave) { - lines.push(`\nThis synthesis will be persisted as a brain page. Aim for completeness — cover Answer, Conflicts, and Gaps thoroughly.`); + lines.push(`\nThis synthesis will be persisted as a brain page. Aim for completeness — cover the Answer and any Conflicts thoroughly, and list every missing piece in the structured "gaps" array.`); } if (opts.withCalibration) { lines.push( @@ -92,7 +92,7 @@ export function buildThinkSystemPrompt(opts: ThinkSystemPromptOpts = {}): string lines.push(`- Name both the user's PRIOR (default reasoning) AND the COUNTER-PRIOR from their hedged-domain self.`); lines.push(`- Reference active bias tags by name when relevant ("this fits the over-confident-geography pattern").`); lines.push(`- Do NOT silently substitute the debiased answer. ALWAYS surface both priors transparently.`); - lines.push(`- Track-record sentences belong in a "Calibration" section in the answer body, between Conflicts and Gaps.`); + lines.push(`- Track-record sentences belong in a "Calibration" section in the answer body, after the Conflicts section (if present).`); } return lines.join('\n'); } diff --git a/test/ai/adaptive-embed-batch.test.ts b/test/ai/adaptive-embed-batch.test.ts index 505c7c82f..144668eca 100644 --- a/test/ai/adaptive-embed-batch.test.ts +++ b/test/ai/adaptive-embed-batch.test.ts @@ -34,6 +34,7 @@ import { resetGateway, embed, splitByTokenBudget, + capBatchItems, isTokenLimitError, __setEmbedTransportForTests, __getShrinkStateForTests, @@ -151,6 +152,41 @@ describe('splitByTokenBudget (pure helper)', () => { }); }); +describe('capBatchItems (hard COUNT cap helper)', () => { + test('batch at or under the cap is returned as a single batch (no copy of contents)', () => { + const texts = ['a', 'b', 'c']; + expect(capBatchItems(texts, 3)).toEqual([texts]); + expect(capBatchItems(texts, 10)).toEqual([texts]); + }); + + test('oversized batch splits into chunks of at most maxItems', () => { + const texts = Array.from({ length: 100 }, (_, i) => `t${i}`); + const result = capBatchItems(texts, 32); + expect(result.map(b => b.length)).toEqual([32, 32, 32, 4]); + expect(result.every(b => b.length <= 32)).toBe(true); + }); + + test('exact multiple splits evenly with no trailing empty batch', () => { + const texts = Array.from({ length: 64 }, (_, i) => `t${i}`); + expect(capBatchItems(texts, 32).map(b => b.length)).toEqual([32, 32]); + }); + + test('order is preserved across the split (concatenation round-trips)', () => { + const texts = Array.from({ length: 70 }, (_, i) => `t${i}`); + expect(capBatchItems(texts, 32).flat()).toEqual(texts); + }); + + test('maxItems <= 0 is a no-op (single batch) — never produces empty/infinite batches', () => { + const texts = ['a', 'b', 'c']; + expect(capBatchItems(texts, 0)).toEqual([texts]); + expect(capBatchItems(texts, -5)).toEqual([texts]); + }); + + test('empty input returns a single empty batch', () => { + expect(capBatchItems([], 32)).toEqual([[]]); + }); +}); + describe('isTokenLimitError (pure helper)', () => { test('matches Voyage error format', () => { expect(isTokenLimitError(VOYAGE_TOKEN_LIMIT_ERROR)).toBe(true); diff --git a/test/ai/no-batch-cap-suppression.serial.test.ts b/test/ai/no-batch-cap-suppression.serial.test.ts index 1241b3b5a..5a5d00f60 100644 --- a/test/ai/no-batch-cap-suppression.serial.test.ts +++ b/test/ai/no-batch-cap-suppression.serial.test.ts @@ -28,8 +28,8 @@ describe('v0.32 #779: no_batch_cap suppresses the missing-max_batch_tokens warni resetGateway(); }); - test('Ollama, LiteLLM, llama-server all declare no_batch_cap: true', () => { - for (const id of ['ollama', 'litellm', 'llama-server']) { + test('Ollama, LiteLLM declare no_batch_cap: true', () => { + for (const id of ['ollama', 'litellm']) { const r = getRecipe(id); expect(r, `${id} not registered`).toBeDefined(); expect( @@ -39,6 +39,16 @@ describe('v0.32 #779: no_batch_cap suppresses the missing-max_batch_tokens warni } }); + test('llama-server declares a hard item-count cap (max_batch_items: 32)', () => { + // llama.cpp enforces a request-COUNT cap equal to its launch --batch-size + // (default 32); declaring max_batch_items both bounds batches AND suppresses + // the missing-max_batch_tokens warning. Replaces the prior no_batch_cap flag. + const r = getRecipe('llama-server'); + expect(r, 'llama-server not registered').toBeDefined(); + expect(r!.touchpoints.embedding?.max_batch_items).toBe(32); + expect(r!.touchpoints.embedding?.no_batch_cap).toBeUndefined(); + }); + test('configureGateway does NOT warn for ollama/litellm/llama-server', () => { warnSpy.mockClear(); resetGateway(); diff --git a/test/ai/recipe-ollama-dims.test.ts b/test/ai/recipe-ollama-dims.test.ts new file mode 100644 index 000000000..4f1d86ce8 --- /dev/null +++ b/test/ai/recipe-ollama-dims.test.ts @@ -0,0 +1,41 @@ +/** + * Ollama Matryoshka dims passthrough. + * + * Several embedding models served via Ollama (Qwen3-Embedding family) support + * Matryoshka truncation through the `dimensions` field on /v1/embeddings. + * Without this passthrough, gbrain ignores user-selected reduced dims and the + * provider returns its native size, causing dim-mismatch failures against + * brains configured for smaller widths. + */ + +import { describe, expect, test } from 'bun:test'; +import { dimsProviderOptions } from '../../src/core/ai/dims.ts'; + +describe('dims: ollama Matryoshka models', () => { + test('qwen3-embedding:4b threads dimensions=1536', () => { + expect(dimsProviderOptions('openai-compatible', 'qwen3-embedding:4b', 1536)) + .toEqual({ openaiCompatible: { dimensions: 1536 } }); + }); + + test('qwen3-embedding:0.6b threads dimensions=512', () => { + expect(dimsProviderOptions('openai-compatible', 'qwen3-embedding:0.6b', 512)) + .toEqual({ openaiCompatible: { dimensions: 512 } }); + }); + + test('qwen3-embedding:8b threads dimensions=2048', () => { + expect(dimsProviderOptions('openai-compatible', 'qwen3-embedding:8b', 2048)) + .toEqual({ openaiCompatible: { dimensions: 2048 } }); + }); + + test('bare qwen3-embedding (no quant tag) also recognized', () => { + expect(dimsProviderOptions('openai-compatible', 'qwen3-embedding', 1024)) + .toEqual({ openaiCompatible: { dimensions: 1024 } }); + }); + + test('unrelated openai-compat model returns undefined (regression guard)', () => { + expect(dimsProviderOptions('openai-compatible', 'nomic-embed-text', 768)) + .toBeUndefined(); + expect(dimsProviderOptions('openai-compatible', 'mxbai-embed-large', 1024)) + .toBeUndefined(); + }); +}); diff --git a/test/autopilot-install.test.ts b/test/autopilot-install.test.ts index 023b03d7d..6355691b1 100644 --- a/test/autopilot-install.test.ts +++ b/test/autopilot-install.test.ts @@ -99,3 +99,29 @@ describe('autopilot wrapper script — env source order (v0.36.1.x #966)', () => expect(src).toMatch(/source\s+~\/\.zshrc/); }); }); + +// v0.42.x: the wrapper must export PATH with ~/.bun/bin before exec'ing +// gbrain. The exec'd gbrain has a `#!/usr/bin/env bun` shebang, and the +// standard Debian ~/.bashrc ships a non-interactive guard +// (`case $- in *i*) ;; *) return;; esac`) that exits early when cron/launchd/ +// systemd invokes bash non-interactively — so the PATH exports that +// operators put in ~/.bashrc never reach this subprocess. Without the +// explicit export the wrapper silently dies with `env: bun: No such file +// or directory`, leaves a stale lockfile, and blocks every subsequent tick +// for the 10-min stale-lock window. Regression: see Hermes `cron doctor` +// reports — this caused a 1-week nightly-cycle outage on at least one +// operator machine before being diagnosed. +describe('autopilot wrapper script — bun PATH export (v0.42.x regression)', () => { + test('wrapper exports ~/.bun/bin onto PATH before the exec', async () => { + const { readFileSync } = await import('fs'); + const src = readFileSync('src/commands/autopilot.ts', 'utf8'); + // The export line must appear inside the writeWrapperScript heredoc. + expect(src).toMatch(/export\s+PATH="\$HOME\/\.bun\/bin:\$PATH"/); + // The export must precede the exec line, otherwise env never sees it. + const exportIdx = src.search(/export\s+PATH="\$HOME\/\.bun\/bin/); + const execIdx = src.search(/exec\s+'\${safeGbrainPath}'/); + expect(exportIdx).toBeGreaterThan(0); + expect(execIdx).toBeGreaterThan(0); + expect(exportIdx).toBeLessThan(execIdx); + }); +}); diff --git a/test/backlinks-job-default.test.ts b/test/backlinks-job-default.test.ts new file mode 100644 index 000000000..399273438 --- /dev/null +++ b/test/backlinks-job-default.test.ts @@ -0,0 +1,47 @@ +/** + * Structural regression for the backlinks Minion handler default. + * + * Backlinks jobs submitted with an EMPTY payload (the sync→embed→backlinks + * chains enqueued after every ingestion) must run as 'check', never 'fix'. + * The pre-fix handler inverted the default (`=== 'check' ? 'check' : 'fix'`), + * so every routine post-ingestion job rewrote tracked brain pages with + * generated "Referenced in" timeline bullets — contradicting the documented + * intent in src/core/cycle.ts (runPhaseBacklinks): "Maintenance cycles must + * not rewrite tracked brain pages with generated 'Referenced in' timeline + * bullets." + * + * Source-grep is the right tool here (see fix-wave-structural.test.ts): the + * handler dynamically imports runBacklinksCore and walks a real repo dir, so + * a behavioral test would require heavy mocking that hides the regression + * behind a test seam. The rule is "this specific default must stay 'check'". + */ +import { describe, test, expect } from 'bun:test'; +import { readFileSync } from 'fs'; + +describe('backlinks Minion handler — empty payload defaults to check, not fix', () => { + const src = readFileSync('src/commands/jobs.ts', 'utf8'); + + // Isolate the backlinks register block so assertions can't accidentally + // match another handler's action parsing. + const blockMatch = src.match( + /worker\.register\('backlinks',[\s\S]*?runBacklinksCore\(\{[\s\S]*?\}\);/ + ); + + test('the backlinks handler block exists', () => { + expect(blockMatch).not.toBeNull(); + }); + + test("default action is 'check' (explicit opt-in required for 'fix')", () => { + const block = blockMatch![0]; + expect(block).toMatch( + /job\.data\.action\s*===\s*'fix'\s*\?\s*'fix'\s*:\s*'check'/ + ); + }); + + test('the inverted (fix-by-default) shape stays absent', () => { + const block = blockMatch![0]; + expect(block).not.toMatch( + /job\.data\.action\s*===\s*'check'\s*\?\s*'check'\s*:\s*'fix'/ + ); + }); +}); diff --git a/test/check-resolvable.test.ts b/test/check-resolvable.test.ts index 7e0c10e1a..b26040b85 100644 --- a/test/check-resolvable.test.ts +++ b/test/check-resolvable.test.ts @@ -6,6 +6,7 @@ import { checkResolvable, parseResolverEntries, extractDelegationTargets, + extractTriggers, } from "../src/core/check-resolvable.ts"; const SKILLS_DIR = join(import.meta.dir, "..", "skills"); @@ -195,6 +196,39 @@ describe("parseResolverEntries", () => { }); }); +describe("extractTriggers", () => { + const LF_FRONTMATTER = + "---\nname: query\ndescription: Test\ntriggers:\n - \"what do we know\"\n - \"tell me about\"\ntools:\n - search\n---\n\n# Body\n"; + + test("parses triggers from LF-terminated frontmatter", () => { + const triggers = extractTriggers(LF_FRONTMATTER); + expect(triggers).toEqual(["what do we know", "tell me about"]); + }); + + test("parses triggers from CRLF-terminated frontmatter (Windows checkouts)", () => { + // Regression: `core.autocrlf=true` is the Windows default. Without + // CRLF→LF normalization, every Windows skill is reported as a false + // mece_gap warning because the `^---\n` regex never matches `---\r\n`. + const crlf = LF_FRONTMATTER.replace(/\n/g, "\r\n"); + const triggers = extractTriggers(crlf); + expect(triggers).toEqual(["what do we know", "tell me about"]); + }); + + test("returns [] when frontmatter is missing", () => { + expect(extractTriggers("# Just a body, no frontmatter\n")).toEqual([]); + }); + + test("returns [] when triggers field is absent from frontmatter", () => { + const fm = "---\nname: query\ndescription: Test\ntools:\n - search\n---\n"; + expect(extractTriggers(fm)).toEqual([]); + }); + + test("strips surrounding quotes from trigger values", () => { + const fm = "---\nname: x\ntriggers:\n - \"double quoted\"\n - 'single quoted'\n - unquoted\n---\n"; + expect(extractTriggers(fm)).toEqual(["double quoted", "single quoted", "unquoted"]); + }); +}); + describe("checkResolvable — real skills directory", () => { const report = checkResolvable(SKILLS_DIR); diff --git a/test/cli-multimodal-integration.test.ts b/test/cli-multimodal-integration.test.ts index 640ecf31c..b4894faad 100644 --- a/test/cli-multimodal-integration.test.ts +++ b/test/cli-multimodal-integration.test.ts @@ -8,13 +8,15 @@ // // PGLite-only: in-memory engine, no DATABASE_URL needed. -import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test'; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from 'bun:test'; import { PGLiteEngine } from '../src/core/pglite-engine.ts'; import { loadConfigWithEngine, type GBrainConfig } from '../src/core/config.ts'; import { + __setRerankTransportForTests, configureGateway, getEmbeddingModel, getMultimodalModel, + rerank, resetGateway, } from '../src/core/ai/gateway.ts'; import type { AIGatewayConfig } from '../src/core/ai/types.ts'; @@ -52,10 +54,16 @@ afterAll(async () => { beforeEach(async () => { resetGateway(); + __setRerankTransportForTests(null); // Clear any prior config rows so tests are independent. setConfig with // empty string is treated as undefined by loadConfigWithEngine (per // dbStr semantics), so this is safe to call between tests. await engine.setConfig('embedding_multimodal_model', ''); + await engine.setConfig('provider_base_urls.llama-server-reranker', ''); +}); + +afterEach(() => { + __setRerankTransportForTests(null); }); describe('cli connectEngine — embedding_multimodal_model DB→gateway plumbing', () => { @@ -122,4 +130,34 @@ describe('cli connectEngine — embedding_multimodal_model DB→gateway plumbing expect(getEmbeddingModel()).toBe('openai:text-embedding-3-large'); expect(getMultimodalModel()).toBeUndefined(); }); + + test('DB-set provider_base_urls.llama-server-reranker flows to gateway.rerank URL', async () => { + await engine.setConfig('provider_base_urls.llama-server-reranker', 'http://127.0.0.1:8091/v1'); + + const baseConfig: GBrainConfig = { + engine: 'pglite', + embedding_model: 'openai:text-embedding-3-large', + embedding_dimensions: 1536, + }; + + const merged = await loadConfigWithEngine(engine, baseConfig); + configureGateway(buildGatewayConfig(merged!)); + + let capturedUrl = ''; + __setRerankTransportForTests(async (url) => { + capturedUrl = url; + return new Response(JSON.stringify({ results: [{ index: 0, relevance_score: 0.9 }] }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }); + + await rerank({ + query: 'q', + documents: ['d'], + model: 'llama-server-reranker:qwen3-reranker-4b', + }); + + expect(capturedUrl).toBe('http://127.0.0.1:8091/v1/rerank'); + }); }); diff --git a/test/cycle/extract-atoms-synthesize-concepts.test.ts b/test/cycle/extract-atoms-synthesize-concepts.test.ts index d14102495..c59e8d7b0 100644 --- a/test/cycle/extract-atoms-synthesize-concepts.test.ts +++ b/test/cycle/extract-atoms-synthesize-concepts.test.ts @@ -345,3 +345,69 @@ describe('v0.41 T6: runPhaseSynthesizeConcepts via stubbed chat', () => { expect((page[0].fm as Record).tier).toBe('T1'); }); }); + +// #2123 — extract_atoms must stamp `concepts` so synthesize_concepts has +// material. The pre-fix pipeline was broken end-to-end: the extractor +// never wrote the field, and every synthesize_concepts cycle skipped with +// "no atoms with concept refs". The earlier describe blocks feed +// synthesize via the `_atoms` seam, which is exactly how the gap survived +// — so the last test here goes extractor → REAL frontmatter → real DB +// query path → concept page. +describe('#2123: concepts label parsing', () => { + test('keeps valid kebab-case labels', () => { + const raw = `[{"title":"T","atom_type":"insight","body":"b","concepts":["captive-portal","tls-certificates"]}]`; + expect(parseAtomsResponse(raw)[0].concepts).toEqual(['captive-portal', 'tls-certificates']); + }); + + test('filters non-kebab labels, keeps the rest', () => { + const raw = `[{"title":"T","atom_type":"insight","body":"b","concepts":["Captive Portal","tp_link","UPPER","valid-label"]}]`; + expect(parseAtomsResponse(raw)[0].concepts).toEqual(['valid-label']); + }); + + test('truncates to 3 labels', () => { + const raw = `[{"title":"T","atom_type":"insight","body":"b","concepts":["a","b","c","d","e"]}]`; + expect(parseAtomsResponse(raw)[0].concepts).toEqual(['a', 'b', 'c']); + }); + + test('absent / non-array / all-invalid → undefined', () => { + expect(parseAtomsResponse(`[{"title":"T","atom_type":"insight","body":"b"}]`)[0].concepts).toBeUndefined(); + expect(parseAtomsResponse(`[{"title":"T","atom_type":"insight","body":"b","concepts":"not-an-array"}]`)[0].concepts).toBeUndefined(); + expect(parseAtomsResponse(`[{"title":"T","atom_type":"insight","body":"b","concepts":["Bad Label!"]}]`)[0].concepts).toBeUndefined(); + }); +}); + +describe('#2123: extractor stamps concepts → synthesize_concepts consumes via real DB path', () => { + test('end-to-end: atoms with shared label materialize a concept page', async () => { + const chat = stubChat(`[ + {"title":"Cert warning on guest wifi","atom_type":"insight","body":"Portal redirects to an IP-based HTTPS URL.","concepts":["captive-portal"]}, + {"title":"iPhone portal popup is flaky","atom_type":"critique","body":"CNA probe behavior differs across iOS versions.","concepts":["captive-portal"]} + ]`); + const extract = await runPhaseExtractAtoms(engine, { + _transcripts: [{ filePath: '/fake/notes.txt', content: 'content', contentHash: 'cc2123' }], + _pages: [], + _chat: chat, + }); + expect(extract.status).toBe('ok'); + expect(extract.details?.atoms_extracted).toBe(2); + + // Frontmatter really carries the label (a jsonb array, not a string). + const stamped = await engine.executeRaw<{ concepts: unknown }>( + `SELECT frontmatter->'concepts' AS concepts FROM pages WHERE type = 'atom'`, + ); + expect(stamped.length).toBe(2); + for (const row of stamped) { + const arr = typeof row.concepts === 'string' ? JSON.parse(row.concepts) : row.concepts; + expect(arr).toEqual(['captive-portal']); + } + + // NO _atoms seam: synthesize discovers the atoms through its own + // DB query — this is the path that was dead before the fix. + const synth = await runPhaseSynthesizeConcepts(engine, { _chat: stubChat('unused — T3 is deterministic') }); + expect(synth.status).toBe('ok'); + expect(synth.details?.concepts_written).toBe(1); + const concept = await engine.executeRaw<{ slug: string }>( + `SELECT slug FROM pages WHERE slug = 'concepts/captive-portal' AND type = 'concept'`, + ); + expect(concept.length).toBe(1); + }); +}); diff --git a/test/doctor-remediation-plan-render.test.ts b/test/doctor-remediation-plan-render.test.ts new file mode 100644 index 000000000..b73d314ff --- /dev/null +++ b/test/doctor-remediation-plan-render.test.ts @@ -0,0 +1,103 @@ +// Regression coverage for the `gbrain doctor --remediation-plan` verdict +// contradiction: when the brain was below target AND the target was +// unreachable, the human renderer printed "Target unreachable: max with +// autonomous remediation is N/100" followed immediately by "No +// remediations needed. Brain is at target." — two consecutive lines that +// contradicted each other and hid the real next step. + +import { describe, test, expect } from 'bun:test'; +import { renderRemediationPlanLines } from '../src/commands/doctor.ts'; + +type Plan = Parameters[0]; + +function planFixture(overrides: Partial): Plan { + return { + brain_score_current: 0, + target_unreachable: false, + max_reachable_score: 100, + plan: [], + est_total_seconds: 0, + est_total_usd_cost: 0, + blocked: [], + ...overrides, + }; +} + +describe('renderRemediationPlanLines', () => { + test('unreachable + brain below target — never claims "Brain is at target"', () => { + const plan = planFixture({ + brain_score_current: 45, + target_unreachable: true, + max_reachable_score: 70, + plan: [], + blocked: [{ check: 'link_density', reason: 'no enrichment keys configured' }], + }); + const text = renderRemediationPlanLines(plan, 90).join('\n'); + expect(text).toContain('Brain score: 45/100'); + expect(text).toContain('Target unreachable: max with autonomous remediation is 70/100'); + expect(text).not.toContain('Brain is at target'); + expect(text).toContain('Blocked checks'); + }); + + test('reachable, brain at or above target, no plan — emits the "at target" line', () => { + const plan = planFixture({ + brain_score_current: 95, + target_unreachable: false, + max_reachable_score: 100, + plan: [], + }); + const text = renderRemediationPlanLines(plan, 90).join('\n'); + expect(text).toContain('Brain is at target'); + expect(text).not.toContain('Target unreachable'); + }); + + test('brain at exact target with empty plan — still "at target"', () => { + const plan = planFixture({ + brain_score_current: 90, + target_unreachable: false, + plan: [], + }); + const text = renderRemediationPlanLines(plan, 90).join('\n'); + expect(text).toContain('Brain is at target'); + }); + + test('brain below target with plan steps — lists the plan, no "at target" line', () => { + const plan = planFixture({ + brain_score_current: 60, + target_unreachable: false, + max_reachable_score: 100, + est_total_seconds: 120, + est_total_usd_cost: 0.4, + plan: [ + { step: 1, severity: 'high', job: 'embed-coverage', rationale: 'missing embeddings' }, + { step: 2, severity: 'med', job: 'consolidate', rationale: 'pending entity merges', est_usd_cost: 0.4 }, + ], + }); + const lines = renderRemediationPlanLines(plan, 90); + const text = lines.join('\n'); + expect(text).toContain('Plan: 2 step(s)'); + expect(text).toContain('1. [high] embed-coverage'); + expect(text).toContain('2. [med] consolidate'); + expect(text).toContain('($0.40)'); + expect(text).not.toContain('Brain is at target'); + }); + + test('unreachable but a partial plan exists — plan prints, "at target" suppressed', () => { + const plan = planFixture({ + brain_score_current: 30, + target_unreachable: true, + max_reachable_score: 55, + est_total_seconds: 90, + est_total_usd_cost: 0.2, + plan: [ + { step: 1, severity: 'high', job: 'embed-coverage', rationale: 'reach max_reachable' }, + ], + blocked: [{ check: 'enrichment', reason: 'no provider key configured' }], + }); + const text = renderRemediationPlanLines(plan, 90).join('\n'); + expect(text).toContain('Target unreachable: max with autonomous remediation is 55/100'); + expect(text).toContain('Plan: 1 step(s)'); + expect(text).toContain('Blocked checks'); + expect(text).not.toContain('Brain is at target'); + }); +}); diff --git a/test/e2e/doctor-progress.test.ts b/test/e2e/doctor-progress.test.ts index 9f3bb7092..f318f52e8 100644 --- a/test/e2e/doctor-progress.test.ts +++ b/test/e2e/doctor-progress.test.ts @@ -20,6 +20,10 @@ import { const skip = !hasDatabase(); const describeE2E = skip ? describe.skip : describe; +if (skip) { + console.log('Skipping E2E doctor --progress-json tests (DATABASE_URL not set)'); +} + const CLI = join(import.meta.dir, '..', '..', 'src', 'cli.ts'); describeE2E('gbrain doctor --progress-json (E2E)', () => { diff --git a/test/e2e/graph-quality.test.ts b/test/e2e/graph-quality.test.ts index 8d99addb6..12a1736df9 100644 --- a/test/e2e/graph-quality.test.ts +++ b/test/e2e/graph-quality.test.ts @@ -29,9 +29,15 @@ afterAll(async () => { }); async function truncateAll() { - for (const t of ['content_chunks', 'links', 'tags', 'raw_data', 'timeline_entries', 'page_versions', 'ingest_log', 'pages']) { + for (const t of ['content_chunks', 'links', 'tags', 'raw_data', 'timeline_entries', 'page_versions', 'ingest_log', 'config', 'pages']) { await (engine as any).db.exec(`DELETE FROM ${t}`); } + // Re-seed the two config keys this file touches back to their documented + // defaults (both default to ON). This makes every test deterministic even if + // an earlier test threw before its finally restored auto_link/auto_timeline, + // and even though absent-key already resolves truthy via isAuto*Enabled. + await engine.setConfig('auto_link', 'true'); + await engine.setConfig('auto_timeline', 'true'); } function makeContext(): OperationContext { @@ -77,10 +83,12 @@ describe('E2E graph quality (v0.10.1 pipeline)', () => { await runExtract(engine, ['links', '--source', 'db']); await runExtract(engine, ['timeline', '--source', 'db']); - // Verify graph populated. + // Verify graph populated. Concrete floors derived from the seeded fixtures: + // resolvable entity refs: alice->acme, bob->acme, standup->alice, standup->bob = 4 + // timeline lines: alice(2) + bob(1) + acme(1) + standup(1) = 5 const stats = await engine.getStats(); - expect(stats.link_count).toBeGreaterThan(0); - expect(stats.timeline_entry_count).toBeGreaterThan(0); + expect(stats.link_count).toBeGreaterThanOrEqual(4); + expect(stats.timeline_entry_count).toBeGreaterThanOrEqual(5); // Verify typed link inference. const aliceLinks = await engine.getLinks('people/alice'); @@ -91,7 +99,16 @@ describe('E2E graph quality (v0.10.1 pipeline)', () => { const bobAcme = bobLinks.find(l => l.to_slug === 'companies/acme'); expect(bobAcme?.link_type).toBe('invested_in'); + // The standup meeting references both Alice and Bob as attendees. Assert the + // exact attendee edges are present and typed 'attended' (a plain .every() + // would silently pass if a meeting->company edge were misclassified or if the + // attendee edges were missing entirely). const meetingLinks = await engine.getLinks('meetings/standup'); + const attended = new Set( + meetingLinks.filter(l => l.link_type === 'attended').map(l => l.to_slug), + ); + expect(attended.has('people/alice')).toBe(true); + expect(attended.has('people/bob')).toBe(true); expect(meetingLinks.every(l => l.link_type === 'attended')).toBe(true); }); @@ -118,7 +135,9 @@ Attendees: [Alice](people/alice). Discussed [Acme](companies/acme). // The response should include auto_links results. expect((result as any).auto_links).toBeDefined(); const autoLinks = (result as any).auto_links; - expect(autoLinks.created).toBeGreaterThan(0); + // The page references exactly two seeded, resolvable targets (Alice + Acme), + // so exactly two links are created. + expect(autoLinks.created).toBe(2); expect(autoLinks.errors).toBe(0); // Verify links actually exist in DB. @@ -283,6 +302,53 @@ Mention of [Alice](people/alice). expect(paths[0].link_type).toBe('works_at'); }); + test('graph-query traversal: direction out and both, plus depth:2 multi-hop', async () => { + // Seed a 2-hop chain: alice -works_at-> acme -partnered_with-> beta. + await engine.putPage('people/alice', { type: 'person', title: 'Alice', compiled_truth: '', timeline: '' }); + await engine.putPage('companies/acme', { type: 'company', title: 'Acme', compiled_truth: '', timeline: '' }); + await engine.putPage('companies/beta', { type: 'company', title: 'Beta', compiled_truth: '', timeline: '' }); + await engine.addLink('people/alice', 'companies/acme', '', 'works_at'); + await engine.addLink('companies/acme', 'companies/beta', '', 'partnered_with'); + + // direction:'out' from alice, depth 1 -> only the first hop. + const out1 = await engine.traversePaths('people/alice', { direction: 'out', depth: 1 }); + expect(out1.length).toBe(1); + expect(out1[0].from_slug).toBe('people/alice'); + expect(out1[0].to_slug).toBe('companies/acme'); + expect(out1[0].depth).toBe(1); + + // depth:2 -> both hops, depths 1 and 2. + const out2 = await engine.traversePaths('people/alice', { direction: 'out', depth: 2 }); + const out2Edges = new Set(out2.map(p => `${p.from_slug}->${p.to_slug}@${p.depth}`)); + expect(out2Edges.has('people/alice->companies/acme@1')).toBe(true); + expect(out2Edges.has('companies/acme->companies/beta@2')).toBe(true); + expect(out2.length).toBe(2); + + // direction:'both' from acme depth 1 -> sees the inbound edge from alice AND + // the outbound edge to beta. Edges keep their natural from->to orientation. + const both = await engine.traversePaths('companies/acme', { direction: 'both', depth: 1 }); + const bothEdges = new Set(both.map(p => `${p.from_slug}->${p.to_slug}`)); + expect(bothEdges.has('people/alice->companies/acme')).toBe(true); + expect(bothEdges.has('companies/acme->companies/beta')).toBe(true); + }); + + test('graph-query cycle safety: A->B->A terminates and returns bounded results', async () => { + await engine.putPage('people/alice', { type: 'person', title: 'Alice', compiled_truth: '', timeline: '' }); + await engine.putPage('people/bob', { type: 'person', title: 'Bob', compiled_truth: '', timeline: '' }); + // Create a 2-cycle: alice -> bob -> alice. + await engine.addLink('people/alice', 'people/bob', '', 'knows'); + await engine.addLink('people/bob', 'people/alice', '', 'knows'); + + // High depth must NOT loop forever; the visited-set guard bounds the walk. + const paths = await engine.traversePaths('people/alice', { direction: 'out', depth: 100 }); + const edges = new Set(paths.map(p => `${p.from_slug}->${p.to_slug}`)); + // Both edges of the cycle are reachable exactly once. + expect(edges.has('people/alice->people/bob')).toBe(true); + expect(edges.has('people/bob->people/alice')).toBe(true); + // Bounded: there are only two edges in the graph, so no path explosion. + expect(paths.length).toBe(2); + }); + test('search backlink boost: well-connected pages rank higher', async () => { // Create 3 pages all matching a search term, but with different inbound link counts. await engine.putPage('topic/popular', { diff --git a/test/e2e/jsonb-roundtrip.test.ts b/test/e2e/jsonb-roundtrip.test.ts index 2b1b69199..9abb276f7 100644 --- a/test/e2e/jsonb-roundtrip.test.ts +++ b/test/e2e/jsonb-roundtrip.test.ts @@ -21,6 +21,10 @@ import { hasDatabase, setupDB, teardownDB, getEngine, getConn } from './helpers. const skip = !hasDatabase(); const describeE2E = skip ? describe.skip : describe; +if (skip) { + console.log('Skipping E2E JSONB roundtrip tests (DATABASE_URL not set)'); +} + describeE2E('E2E: JSONB roundtrip — v0.12.1 reliability wave', () => { beforeAll(async () => { await setupDB(); }); afterAll(async () => { await teardownDB(); }); diff --git a/test/e2e/mcp.test.ts b/test/e2e/mcp.test.ts index 530f3c036..1f985514a 100644 --- a/test/e2e/mcp.test.ts +++ b/test/e2e/mcp.test.ts @@ -56,6 +56,7 @@ describe('E2E: MCP Tool Generation', () => { expect(names).toContain('get_health'); expect(names).toContain('sync_brain'); expect(names).toContain('file_upload'); + expect(names).toContain('find_orphans'); }); test('MCP server module can be imported', async () => { diff --git a/test/e2e/mechanical.test.ts b/test/e2e/mechanical.test.ts index 8d6c52568..3b21e1e92 100644 --- a/test/e2e/mechanical.test.ts +++ b/test/e2e/mechanical.test.ts @@ -175,6 +175,15 @@ describeE2E('E2E: Search', () => { for (const [query, score] of Object.entries(scores)) { console.log(` "${query}": ${(score * 100).toFixed(0)}%`); } + + // Guard value: every known-item query must surface at least one ground-truth + // doc in the top 5. This is a deliberately loose floor (not a tuned P@5 + // threshold) — it catches a total keyword-retrieval regression without + // breaking on every scoring/fixture tweak. Without it this test asserted + // nothing and a 0%-precision result passed silently. + for (const [query, score] of Object.entries(scores)) { + expect(score).toBeGreaterThan(0); + } }); }); @@ -205,10 +214,22 @@ describeE2E('E2E: Links', () => { }, 30_000); test('traverse_graph finds connected pages', async () => { - // Links should already be added from prior test in this describe block - const graph = await callOp('traverse_graph', { slug: 'people/sarah-chen', depth: 2 }) as any; + // Self-contained: do not depend on a prior test's add_link. add_link is + // idempotent (ON CONFLICT DO NOTHING), so re-adding here is safe whether or + // not the round-trip test ran first, and the test no longer false-passes or + // false-fails based on describe-block ordering. + await callOp('add_link', { + from: 'people/sarah-chen', + to: 'companies/novamind', + link_type: 'founded', + }); + + const graph = await callOp('traverse_graph', { slug: 'people/sarah-chen', depth: 2 }) as any[]; expect(Array.isArray(graph)).toBe(true); expect(graph.length).toBeGreaterThanOrEqual(1); + // Content assertion, not just shape: the linked company must be reachable. + const reachable = graph.map((n: any) => n.slug ?? n.to_slug ?? n.to_page_slug); + expect(reachable).toContain('companies/novamind'); }); test('remove_link removes the link', async () => { @@ -469,8 +490,14 @@ describeE2E('E2E: Admin', () => { test('get_health returns valid structure', async () => { const health = await callOp('get_health') as any; expect(health).toBeDefined(); - expect(typeof health.page_count).toBe('number'); - expect(typeof health.embed_coverage).toBe('number'); + // Value bounds, not just types: page_count must match the fixture inventory + // and embed_coverage is a 0..1 fraction (src/commands/doctor.ts multiplies + // by 100 and compares to 0.9). Type-only checks let embed_coverage: -9999 + // through; these catch a genuinely broken health payload. + expect(health.page_count).toBe(16); + expect(Number.isFinite(health.embed_coverage)).toBe(true); + expect(health.embed_coverage).toBeGreaterThanOrEqual(0); + expect(health.embed_coverage).toBeLessThanOrEqual(1); }); }); @@ -488,7 +515,17 @@ describeE2E('E2E: Chunks & Resolution', () => { test('get_chunks returns chunks for imported page', async () => { const chunks = await callOp('get_chunks', { slug: 'people/sarah-chen' }) as any[]; expect(chunks.length).toBeGreaterThan(0); - expect(chunks[0].chunk_text).toBeTruthy(); + // Content + ordering, not just truthiness (a whitespace-only chunk is truthy): + // every chunk has real text and a numeric index, the indexes are + // non-decreasing in return order, and the page's own name appears somewhere. + for (const c of chunks) { + expect(typeof c.chunk_text).toBe('string'); + expect(c.chunk_text.trim().length).toBeGreaterThan(0); + expect(typeof c.chunk_index).toBe('number'); + } + const indexes = chunks.map((c: any) => c.chunk_index); + expect(indexes).toEqual([...indexes].sort((x, y) => x - y)); + expect(chunks.some((c: any) => c.chunk_text.includes('Sarah'))).toBe(true); }, 30_000); test('resolve_slugs finds partial match', async () => { @@ -662,9 +699,29 @@ describeE2E('E2E: file_list LIMIT enforcement', () => { }, 30_000); test('file_list without slug also respects LIMIT 100', async () => { - // The 150 rows from the previous test are still in the DB + // Self-sufficient: seed our own >100 rows rather than relying on the + // previous test's 150 rows surviving in the DB. A bun reorder, a focused + // `-t` run, or a failure mid-insert in the prior test would otherwise leave + // this asserting against an indeterminate row count. + const sql = getConn(); + const seedSlug = 'test-limit-noslug'; + await sql` + INSERT INTO pages (slug, title, type, compiled_truth, frontmatter) + VALUES (${seedSlug}, ${'Test Limit NoSlug'}, ${'note'}, ${'body'}, ${'{}'}::jsonb) + ON CONFLICT (source_id, slug) DO NOTHING + `; + for (let i = 0; i < 120; i++) { + await sql` + INSERT INTO files (page_slug, filename, storage_path, mime_type, size_bytes, content_hash, metadata) + VALUES (${seedSlug}, ${'nf-' + String(i).padStart(3, '0') + '.txt'}, ${seedSlug + '/nf-' + i + '.txt'}, ${'text/plain'}, ${100}, ${'nhash-' + i}, ${'{}'}::jsonb) + ON CONFLICT (storage_path) DO NOTHING + `; + } + const total = await sql`SELECT count(*)::int AS n FROM files`; + expect(Number(total[0].n)).toBeGreaterThan(100); // cap is actually exercised + const files = await callOp('file_list', {}) as any[]; - expect(files.length).toBeLessThanOrEqual(100); + expect(files.length).toBe(100); }); }); diff --git a/test/e2e/migration-flow.test.ts b/test/e2e/migration-flow.test.ts index 7ba06eebf..c0726e4be 100644 --- a/test/e2e/migration-flow.test.ts +++ b/test/e2e/migration-flow.test.ts @@ -78,6 +78,19 @@ function freshTempHome(label: string) { return dir; } +// Restore HOME/PATH to the captured originals. Called from each test's +// `finally` so a throw mid-test can never leave HOME/PATH pointed at a temp +// dir for the rest of the bun process (which would silently break unrelated +// suites that read HOME). PATH keeps the shim prepended because the +// module-level shim install is what subsequent tests in this suite rely on; +// afterAll does the final teardown to the pristine origPath. +function restoreHomePath() { + if (origHome === undefined) delete process.env.HOME; + else process.env.HOME = origHome; + if (origPath === undefined) delete process.env.PATH; + else process.env.PATH = `${fakeBinDir}:${origPath ?? ''}`; +} + beforeAll(() => { if (SKIP) { console.log('[migration-flow.e2e] DATABASE_URL not set — skipping.'); @@ -100,6 +113,15 @@ afterAll(() => { beforeEach(() => { if (SKIP) return; + // Robust restore: if a prior test threw before its own finally ran (or + // before afterAll), HOME/PATH could still point at a dead temp dir. Reset + // them to the captured originals at the start of every test so a throw in + // one test can never leak a temp HOME/PATH into sibling suites that read + // them. freshTempHome() re-points HOME per test immediately after this. + if (origHome === undefined) delete process.env.HOME; + else process.env.HOME = origHome; + if (origPath === undefined) delete process.env.PATH; + else process.env.PATH = `${fakeBinDir}:${origPath ?? ''}`; try { if (tmp) rmSync(tmp, { recursive: true, force: true }); } catch { /* best-effort */ } }); @@ -114,144 +136,160 @@ const COMMON_OPTS = { describeE2E('E2E: v0.11.0 orchestrator against live Postgres', () => { test('fresh install flow: schema → smoke → prefs → host-rewrite → completed', async () => { tmp = freshTempHome('fresh'); - const result = await v0_11_0.orchestrator(COMMON_OPTS); + try { + const result = await v0_11_0.orchestrator(COMMON_OPTS); - // Orchestrator returns a structured result (status is `complete` when - // no pending-host-work TODOs fired, `partial` otherwise). - expect(result.version).toBe('0.11.0'); - expect(['complete', 'partial']).toContain(result.status); + // Orchestrator returns a structured result (status is `complete` when + // no pending-host-work TODOs fired, `partial` otherwise). + expect(result.version).toBe('0.11.0'); + expect(['complete', 'partial']).toContain(result.status); - // Phase D: preferences.json exists with 0o600 + mode=pain_triggered. - const prefsPath = join(tmp, '.gbrain', 'preferences.json'); - expect(existsSync(prefsPath)).toBe(true); - expect(statSync(prefsPath).mode & 0o777).toBe(0o600); - const prefs = loadPreferences(); - expect(prefs.minion_mode).toBe('pain_triggered'); - expect(prefs.set_at).toBeTruthy(); - expect(prefs.set_in_version).toBeTruthy(); + // Phase D: preferences.json exists with 0o600 + mode=pain_triggered. + const prefsPath = join(tmp, '.gbrain', 'preferences.json'); + expect(existsSync(prefsPath)).toBe(true); + expect(statSync(prefsPath).mode & 0o777).toBe(0o600); + const prefs = loadPreferences(); + expect(prefs.minion_mode).toBe('pain_triggered'); + expect(prefs.set_at).toBeTruthy(); + expect(prefs.set_in_version).toBeTruthy(); - // Bug 3 (v0.14.2) — orchestrator no longer writes completed.jsonl. - // The runner (apply-migrations.ts) persists the result after the - // orchestrator returns. A direct orchestrator call in E2E leaves the - // ledger empty; the runner path is tested separately in - // test/apply-migrations.test.ts + test/migration-resume.test.ts. - const completed = loadCompletedMigrations(); - const v0110Entries = completed.filter(e => e.version === '0.11.0'); - expect(v0110Entries.length).toBe(0); + // Bug 3 (v0.14.2) — orchestrator no longer writes completed.jsonl. + // The runner (apply-migrations.ts) persists the result after the + // orchestrator returns. A direct orchestrator call in E2E leaves the + // ledger empty; the runner path is tested separately in + // test/apply-migrations.test.ts + test/migration-resume.test.ts. + const completed = loadCompletedMigrations(); + const v0110Entries = completed.filter(e => e.version === '0.11.0'); + expect(v0110Entries.length).toBe(0); - // Phase F is skipped per COMMON_OPTS — autopilot should NOT have been - // installed on this host. - expect(result.autopilot_installed).toBe(false); + // Phase F is skipped per COMMON_OPTS — autopilot should NOT have been + // installed on this host. + expect(result.autopilot_installed).toBe(false); + } finally { + restoreHomePath(); + } }, 60_000); test('idempotent rerun: second invocation is a safe no-op', async () => { tmp = freshTempHome('rerun'); - const first = await v0_11_0.orchestrator(COMMON_OPTS); - expect(['complete', 'partial']).toContain(first.status); + try { + const first = await v0_11_0.orchestrator(COMMON_OPTS); + expect(['complete', 'partial']).toContain(first.status); - const second = await v0_11_0.orchestrator(COMMON_OPTS); - expect(['complete', 'partial']).toContain(second.status); + const second = await v0_11_0.orchestrator(COMMON_OPTS); + expect(['complete', 'partial']).toContain(second.status); - // Bug 3 (v0.14.2) — orchestrator does not write completed.jsonl, so - // repeated direct invocations don't accumulate ledger entries. Assert - // the preferences state stays stable (the real idempotency signal for - // this orchestrator is "running again doesn't corrupt preferences"). - expect(loadPreferences().minion_mode).toBe('pain_triggered'); - const completed = loadCompletedMigrations(); - expect(completed.filter(e => e.version === '0.11.0').length).toBe(0); + // Bug 3 (v0.14.2) — orchestrator does not write completed.jsonl, so + // repeated direct invocations don't accumulate ledger entries. Assert + // the preferences state stays stable (the real idempotency signal for + // this orchestrator is "running again doesn't corrupt preferences"). + expect(loadPreferences().minion_mode).toBe('pain_triggered'); + const completed = loadCompletedMigrations(); + expect(completed.filter(e => e.version === '0.11.0').length).toBe(0); + } finally { + restoreHomePath(); + } }, 90_000); test('host rewrite: builtin handlers auto-rewritten, non-builtins queued as JSONL TODOs', async () => { tmp = freshTempHome('host-rewrite'); - // Fixture: AGENTS.md + cron/jobs.json with a mix of gbrain-builtin and - // non-builtin handlers. - const claudeDir = join(tmp, '.claude'); - mkdirSync(claudeDir, { recursive: true }); - writeFileSync( - join(claudeDir, 'AGENTS.md'), - '# Test AGENTS.md\n\nSome existing content referencing sessions_spawn routing.\n', - ); - mkdirSync(join(claudeDir, 'cron'), { recursive: true }); - writeFileSync( - join(claudeDir, 'cron', 'jobs.json'), - JSON.stringify({ - jobs: [ - { schedule: '*/5 * * * *', kind: 'agentTurn', skill: 'sync' }, // builtin - { schedule: '0 */30 * * *', kind: 'agentTurn', skill: 'ea-inbox-sweep' }, // non-builtin - { schedule: '*/10 * * * *', kind: 'agentTurn', skill: 'embed' }, // builtin - { schedule: '0 8 * * *', kind: 'agentTurn', skill: 'morning-briefing' }, // non-builtin - ], - }, null, 2) + '\n', - ); + try { + // Fixture: AGENTS.md + cron/jobs.json with a mix of gbrain-builtin and + // non-builtin handlers. + const claudeDir = join(tmp, '.claude'); + mkdirSync(claudeDir, { recursive: true }); + writeFileSync( + join(claudeDir, 'AGENTS.md'), + '# Test AGENTS.md\n\nSome existing content referencing sessions_spawn routing.\n', + ); + mkdirSync(join(claudeDir, 'cron'), { recursive: true }); + writeFileSync( + join(claudeDir, 'cron', 'jobs.json'), + JSON.stringify({ + jobs: [ + { schedule: '*/5 * * * *', kind: 'agentTurn', skill: 'sync' }, // builtin + { schedule: '0 */30 * * *', kind: 'agentTurn', skill: 'ea-inbox-sweep' }, // non-builtin + { schedule: '*/10 * * * *', kind: 'agentTurn', skill: 'embed' }, // builtin + { schedule: '0 8 * * *', kind: 'agentTurn', skill: 'morning-briefing' }, // non-builtin + ], + }, null, 2) + '\n', + ); - const result = await v0_11_0.orchestrator(COMMON_OPTS); + const result = await v0_11_0.orchestrator(COMMON_OPTS); - // Builtins rewritten in place; non-builtins left alone. - const cronAfter = JSON.parse(readFileSync(join(claudeDir, 'cron', 'jobs.json'), 'utf-8')); - expect(cronAfter.jobs[0].kind).toBe('shell'); // sync (builtin) - expect(cronAfter.jobs[0].cmd).toContain('gbrain jobs submit sync'); - expect(cronAfter.jobs[1].kind).toBe('agentTurn'); // ea-inbox-sweep (non-builtin) - expect(cronAfter.jobs[2].kind).toBe('shell'); // embed (builtin) - expect(cronAfter.jobs[3].kind).toBe('agentTurn'); // morning-briefing (non-builtin) + // Builtins rewritten in place; non-builtins left alone. + const cronAfter = JSON.parse(readFileSync(join(claudeDir, 'cron', 'jobs.json'), 'utf-8')); + expect(cronAfter.jobs[0].kind).toBe('shell'); // sync (builtin) + expect(cronAfter.jobs[0].cmd).toContain('gbrain jobs submit sync'); + expect(cronAfter.jobs[1].kind).toBe('agentTurn'); // ea-inbox-sweep (non-builtin) + expect(cronAfter.jobs[2].kind).toBe('shell'); // embed (builtin) + expect(cronAfter.jobs[3].kind).toBe('agentTurn'); // morning-briefing (non-builtin) - // files_rewritten counts the 2 builtin rewrites. - expect(result.files_rewritten).toBeGreaterThanOrEqual(2); + // files_rewritten counts the 2 builtin rewrites. + expect(result.files_rewritten).toBeGreaterThanOrEqual(2); - // pending_host_work counts the 2 non-builtin TODOs. - expect(result.pending_host_work).toBe(2); + // pending_host_work counts the 2 non-builtin TODOs. + expect(result.pending_host_work).toBe(2); - // Status is "partial" because non-builtin TODOs remain. - expect(result.status).toBe('partial'); + // Status is "partial" because non-builtin TODOs remain. + expect(result.status).toBe('partial'); - // AGENTS.md got the marker injected. - const agentsMdAfter = readFileSync(join(claudeDir, 'AGENTS.md'), 'utf-8'); - expect(agentsMdAfter).toContain('gbrain:subagent-routing v0.11.0'); - expect(agentsMdAfter).toContain('skills/conventions/subagent-routing.md'); + // AGENTS.md got the marker injected. + const agentsMdAfter = readFileSync(join(claudeDir, 'AGENTS.md'), 'utf-8'); + expect(agentsMdAfter).toContain('gbrain:subagent-routing v0.11.0'); + expect(agentsMdAfter).toContain('skills/conventions/subagent-routing.md'); - // JSONL TODO file written under ~/.gbrain/migrations/. - const jsonlPath = join(tmp, '.gbrain', 'migrations', 'pending-host-work.jsonl'); - expect(existsSync(jsonlPath)).toBe(true); - const lines = readFileSync(jsonlPath, 'utf-8').split('\n').filter(l => l.trim()); - expect(lines.length).toBe(2); - const todos = lines.map(l => JSON.parse(l)); - const handlers = todos.map(t => t.handler).sort(); - expect(handlers).toEqual(['ea-inbox-sweep', 'morning-briefing']); - for (const todo of todos) { - expect(todo.type).toBe('cron-handler-needs-host-registration'); - expect(todo.status).toBe('pending'); - expect(todo.manifest_path).toContain('cron/jobs.json'); + // JSONL TODO file written under ~/.gbrain/migrations/. + const jsonlPath = join(tmp, '.gbrain', 'migrations', 'pending-host-work.jsonl'); + expect(existsSync(jsonlPath)).toBe(true); + const lines = readFileSync(jsonlPath, 'utf-8').split('\n').filter(l => l.trim()); + expect(lines.length).toBe(2); + const todos = lines.map(l => JSON.parse(l)); + const handlers = todos.map(t => t.handler).sort(); + expect(handlers).toEqual(['ea-inbox-sweep', 'morning-briefing']); + for (const todo of todos) { + expect(todo.type).toBe('cron-handler-needs-host-registration'); + expect(todo.status).toBe('pending'); + expect(todo.manifest_path).toContain('cron/jobs.json'); + } + } finally { + restoreHomePath(); } }, 90_000); test('resumable: partial run → orchestrator re-run → complete', async () => { tmp = freshTempHome('resumable'); - // Simulate a stopgap-written partial entry BEFORE running the orchestrator. - mkdirSync(join(tmp, '.gbrain', 'migrations'), { recursive: true }); - writeFileSync( - join(tmp, '.gbrain', 'migrations', 'completed.jsonl'), - JSON.stringify({ - version: '0.11.0', - status: 'partial', - apply_migrations_pending: true, - mode: 'pain_triggered', - source: 'fix-v0.11.0.sh', - ts: new Date().toISOString(), - }) + '\n', - ); + try { + // Simulate a stopgap-written partial entry BEFORE running the orchestrator. + mkdirSync(join(tmp, '.gbrain', 'migrations'), { recursive: true }); + writeFileSync( + join(tmp, '.gbrain', 'migrations', 'completed.jsonl'), + JSON.stringify({ + version: '0.11.0', + status: 'partial', + apply_migrations_pending: true, + mode: 'pain_triggered', + source: 'fix-v0.11.0.sh', + ts: new Date().toISOString(), + }) + '\n', + ); - // Orchestrator re-running on a partial → should succeed (schema apply - // and smoke are idempotent; prefs are preserved from the partial - // record; host-rewrite runs its safe-skip pass). Per Bug 3 (v0.14.2), - // the orchestrator itself doesn't append to completed.jsonl — the - // runner does. The stopgap's partial entry stays unchanged here. - const result = await v0_11_0.orchestrator(COMMON_OPTS); - expect(['complete', 'partial']).toContain(result.status); + // Orchestrator re-running on a partial → should succeed (schema apply + // and smoke are idempotent; prefs are preserved from the partial + // record; host-rewrite runs its safe-skip pass). Per Bug 3 (v0.14.2), + // the orchestrator itself doesn't append to completed.jsonl — the + // runner does. The stopgap's partial entry stays unchanged here. + const result = await v0_11_0.orchestrator(COMMON_OPTS); + expect(['complete', 'partial']).toContain(result.status); - const completed = loadCompletedMigrations(); - const v0110 = completed.filter(e => e.version === '0.11.0'); - // Just the stopgap partial — orchestrator doesn't add its own entry. - expect(v0110.length).toBe(1); - expect(v0110[0].status).toBe('partial'); - expect(v0110[0].source).toBe('fix-v0.11.0.sh'); + const completed = loadCompletedMigrations(); + const v0110 = completed.filter(e => e.version === '0.11.0'); + // Just the stopgap partial — orchestrator doesn't add its own entry. + expect(v0110.length).toBe(1); + expect(v0110[0].status).toBe('partial'); + expect(v0110[0].source).toBe('fix-v0.11.0.sh'); + } finally { + restoreHomePath(); + } }, 90_000); }); diff --git a/test/e2e/minions-resilience.test.ts b/test/e2e/minions-resilience.test.ts index 21614fdd5..bd1ce0419 100644 --- a/test/e2e/minions-resilience.test.ts +++ b/test/e2e/minions-resilience.test.ts @@ -94,7 +94,7 @@ describeE2E('E2E: Minions resilience (OpenClaw real-world patterns)', () => { }, 30_000); // --- 2. Runaway handler: ignores AbortSignal, dead-lettered by handleTimeouts --- - test('runaway handler: ignores AbortSignal, handleTimeouts dead-letters in <2s', async () => { + test('runaway handler: ignores AbortSignal, handleTimeouts dead-letters it', async () => { const { a, b } = await makeEngines(); try { const queue = new MinionQueue(a); @@ -133,8 +133,14 @@ describeE2E('E2E: Minions resilience (OpenClaw real-world patterns)', () => { worker.stop(); await startP; + // Correctness gate: the job MUST be dead-lettered with the timeout reason. + // We intentionally do NOT assert a wall-clock upper bound (deadAt - started): + // on a loaded CI runner the stall/timeout sweep cadence varies, and the only + // thing that matters is that the runaway job terminates as 'dead'. The 3s poll + // deadline above is the real timeout — if the sweep is too slow, finalStatus + // stays '' and this toBe('dead') fails loudly. expect(finalStatus).toBe('dead'); - expect(deadAt - started).toBeLessThan(2000); + void deadAt; // retained for debugging; no timing assertion (flake-prone) const final = await queue.getJob(job.id); expect(final?.error_text).toMatch(/timeout exceeded/i); @@ -304,7 +310,7 @@ describeE2E('E2E: Minions resilience (OpenClaw real-world patterns)', () => { }, 60_000); // --- 5. Cascade kill under load: cancelJob aborts all live descendants --- - test('cascade kill: cancelJob on parent aborts 10 live children within 2s', async () => { + test('cascade kill: cancelJob on parent aborts 10 live children', async () => { const { a, b } = await makeEngines(); try { const queue = new MinionQueue(a); @@ -374,8 +380,12 @@ describeE2E('E2E: Minions resilience (OpenClaw real-world patterns)', () => { worker.stop(); await startP; + // Correctness gate: all 10 cooperative handlers observed the abort and the + // DB shows every descendant + root cancelled. We do NOT assert a wall-clock + // upper bound on cancelElapsed — the 3s abort poll deadline above already + // bounds the wait, and asserting a tighter time flakes on shared runners. expect(abortedChildren.size).toBe(10); - expect(cancelElapsed).toBeLessThan(3000); + void cancelElapsed; // retained for debugging; no timing assertion (flake-prone) // DB truth: every descendant + root is 'cancelled' const conn = getConn(); diff --git a/test/e2e/multi-source.test.ts b/test/e2e/multi-source.test.ts index f30bef121..5c45c6b7a 100644 --- a/test/e2e/multi-source.test.ts +++ b/test/e2e/multi-source.test.ts @@ -78,7 +78,11 @@ describeE2E('v0.18.0 multi-source — Postgres schema shape (fresh install)', () ); expect(rows.length).toBe(1); expect(rows[0].is_nullable).toBe('NO'); - expect(String(rows[0].column_default)).toContain('default'); + // Postgres renders a TEXT DEFAULT 'default' literal as `'default'::text`. + // Assert the exact stored expression rather than a loose substring so a + // drift in the schema DEFAULT (e.g. a different sentinel source id) fails + // here instead of silently passing. + expect(String(rows[0].column_default)).toBe("'default'::text"); }); test('composite UNIQUE pages(source_id, slug) replaces global UNIQUE(slug)', async () => { @@ -292,6 +296,18 @@ describeE2E('v0.18.0 multi-source — cascade delete covers every dependent row' `INSERT INTO files (source_id, page_id, filename, storage_path, content_hash) VALUES ('cascadetest', ${aliceId}, 'alice.pdf', 'cascadetest/people/alice/alice.pdf', 'fh1')`, ); + const aliceFile = await conn.unsafe( + `SELECT id FROM files WHERE source_id = 'cascadetest' AND storage_path = 'cascadetest/people/alice/alice.pdf'`, + ); + const aliceFileId = aliceFile[0].id as number; + + // file_migration_ledger row keyed on the file (FK file_id ON DELETE + // CASCADE). Removing the source cascades sources → files → ledger. + await conn.unsafe( + `INSERT INTO file_migration_ledger (file_id, storage_path_old, storage_path_new, status) + VALUES (${aliceFileId}, 'cascadetest/people/alice/alice.pdf', 'cascadetest/people/alice/alice.pdf', 'pending') + ON CONFLICT (file_id) DO NOTHING`, + ); // Sanity: everything exists expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM pages WHERE source_id = 'cascadetest'`))[0].n).toBe(2); @@ -299,6 +315,7 @@ describeE2E('v0.18.0 multi-source — cascade delete covers every dependent row' expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM timeline_entries WHERE page_id = ${aliceId}`))[0].n).toBe(1); expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM links WHERE from_page_id = ${aliceId}`))[0].n).toBe(1); expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM files WHERE source_id = 'cascadetest'`))[0].n).toBe(1); + expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM file_migration_ledger WHERE file_id = ${aliceFileId}`))[0].n).toBe(1); // Remove the source. // v0.26.5: populated sources require --confirm-destructive; --yes alone is rejected. @@ -310,6 +327,7 @@ describeE2E('v0.18.0 multi-source — cascade delete covers every dependent row' expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM timeline_entries WHERE page_id = ${aliceId}`))[0].n).toBe(0); expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM links WHERE from_page_id = ${aliceId}`))[0].n).toBe(0); expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM files WHERE source_id = 'cascadetest'`))[0].n).toBe(0); + expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM file_migration_ledger WHERE file_id = ${aliceFileId}`))[0].n).toBe(0); // The sources row itself is gone. const src = await conn.unsafe(`SELECT id FROM sources WHERE id = 'cascadetest'`); @@ -378,8 +396,10 @@ describeE2E('v0.18.0 multi-source — sync --source routes through sources table test('performSync with no sourceId falls back to global sync.repo_path', async () => { const engine = getEngine(); - // Global config is still '/some/other/default/path' from the - // previous test. Without --source, performSync uses it. + // Self-contained: set the global config this test depends on directly + // instead of inheriting the side effect of the previous test. Without + // --source, performSync must read this global key. + await engine.setConfig('sync.repo_path', '/some/other/default/path'); let err: Error | null = null; try { await performSync(engine, {}); diff --git a/test/e2e/salience-pglite.test.ts b/test/e2e/salience-pglite.test.ts index aa959031e..98d46604a 100644 --- a/test/e2e/salience-pglite.test.ts +++ b/test/e2e/salience-pglite.test.ts @@ -112,4 +112,27 @@ describe('v0.29 E2E — getRecentSalience (Garry test)', () => { const rows = await engine.getRecentSalience({ days: 7, slugPrefix: 'nope/does-not-exist/' }); expect(rows).toEqual([]); }); + + // TIM-37: the daily briefing writes to the vault and re-ingests as + // `briefings/`. Without this filter the briefing itself would top + // every subsequent Brain Pulse — self-reference with no signal. + describe('TIM-37 — briefings excluded from their own Brain Pulse', () => { + test('default query hides briefings/* slugs', async () => { + await engine.putPage('briefings/2026-05-19', { + type: 'note', + title: 'Daily Briefing — 2026-05-19', + compiled_truth: 'Auto-generated cron briefing.', + }); + const rows = await engine.getRecentSalience({ days: 7, limit: 50 }); + expect(rows.some(r => r.slug.startsWith('briefings/'))).toBe(false); + }); + + test('explicit slugPrefix=briefings/ still returns them', async () => { + const rows = await engine.getRecentSalience({ days: 7, slugPrefix: 'briefings/' }); + expect(rows.length).toBeGreaterThan(0); + for (const r of rows) { + expect(r.slug.startsWith('briefings/')).toBe(true); + } + }); + }); }); diff --git a/test/e2e/search-quality.test.ts b/test/e2e/search-quality.test.ts index 3b2e73a77..6c61d8293 100644 --- a/test/e2e/search-quality.test.ts +++ b/test/e2e/search-quality.test.ts @@ -128,6 +128,17 @@ describe('SearchResult fields', () => { expect(r.chunk_index).toBeDefined(); expect(typeof r.chunk_index).toBe('number'); }); + + test('empty keyword query returns a defined array without throwing', async () => { + const results = await engine.searchKeyword(''); + expect(Array.isArray(results)).toBe(true); + }); + + test('zero vector search returns a defined array without throwing', async () => { + const zeroVector = new Float32Array(1536); + const results = await engine.searchVector(zeroVector); + expect(Array.isArray(results)).toBe(true); + }); }); describe('detail parameter', () => { @@ -145,9 +156,11 @@ describe('detail parameter', () => { }); test('detail=low on vector search filters to compiled_truth', async () => { - // Use a timeline-direction embedding — with detail=low, should get no results - // or only compiled_truth results + // Use a timeline-direction embedding — detail=low filters to compiled_truth. + // Vector search returns every chunk with an embedding (ordered by distance), + // so the seeded compiled_truth chunks are non-empty and ALL compiled_truth. const results = await engine.searchVector(basisEmbedding(1), { detail: 'low' }); + expect(results.length).toBeGreaterThan(0); for (const r of results) { expect(r.chunk_source).toBe('compiled_truth'); } diff --git a/test/e2e/upgrade.test.ts b/test/e2e/upgrade.test.ts index ecbe31e35..af204e489 100644 --- a/test/e2e/upgrade.test.ts +++ b/test/e2e/upgrade.test.ts @@ -73,7 +73,7 @@ describeE2E('E2E: Check-Update', () => { expect(stdout).toContain('--json'); }); - test('handles no-releases gracefully (current repo state)', async () => { + test('check-update --json contract holds regardless of real release state', async () => { const proc = Bun.spawn(['bun', 'run', 'src/cli.ts', 'check-update', '--json'], { cwd: new URL('../..', import.meta.url).pathname, stdout: 'pipe', @@ -84,8 +84,16 @@ describeE2E('E2E: Check-Update', () => { expect(exitCode).toBe(0); const output = JSON.parse(stdout); - // With no releases, should return false and an error - expect(output.update_available).toBe(false); + // Don't pin update_available to a literal value — the repo may or may not + // have a published release. Assert the JSON shape instead. + expect(typeof output.update_available).toBe('boolean'); + expect(output.current_version).toBe(VERSION); + if (output.latest_version != null) { + expect(typeof output.latest_version).toBe('string'); + } + if (output.release_url != null) { + expect(typeof output.release_url).toBe('string'); + } }); test('version comparison wiring works end-to-end', () => { diff --git a/test/extract-atoms-drain.test.ts b/test/extract-atoms-drain.test.ts index cb7dae821..ed316ecc9 100644 --- a/test/extract-atoms-drain.test.ts +++ b/test/extract-atoms-drain.test.ts @@ -134,3 +134,42 @@ describe('shared wiring helper holds the cycle lock (5A)', () => { expect(src).toContain('withRefreshingLock(engine, lockId'); }); }); + +describe('#2144: zero-yield tombstone progress semantics', () => { + it('continues when a zero-atom batch still shrinks the backlog (tombstoned pages)', async () => { + let batches = 0; + const result = await runExtractAtomsDrain( + { + withLock: passThroughLock, + // consumed: before#1=4, after#1=2 (<4 → progress), before#2=2, + // after#2=0 (<2 → progress), before#3=0 → drained; final repeats 0. + countRemaining: seq([4, 2, 2, 0, 0]), + runBatch: async () => { batches++; return { extracted: 0, skipped: 0 }; }, + now: () => 0, + }, + { windowMs: 1_000_000 }, + ); + expect(result.stopped).toBe('drained'); + expect(result.batches).toBe(2); + expect(result.extracted).toBe(0); + expect(result.remaining).toBe(0); + expect(batches).toBe(2); + }); + + it('stops no_progress when a zero-atom batch leaves the backlog flat', async () => { + let batches = 0; + const result = await runExtractAtomsDrain( + { + withLock: passThroughLock, + countRemaining: seq([5, 5]), + runBatch: async () => { batches++; return { extracted: 0, skipped: 0 }; }, + now: () => 0, + }, + { windowMs: 1_000_000 }, + ); + expect(result.stopped).toBe('no_progress'); + expect(result.batches).toBe(1); + expect(result.remaining).toBe(5); + expect(batches).toBe(1); + }); +}); diff --git a/test/extract-atoms-page-discovery.test.ts b/test/extract-atoms-page-discovery.test.ts index 7289cc52e..4a0f7b061 100644 --- a/test/extract-atoms-page-discovery.test.ts +++ b/test/extract-atoms-page-discovery.test.ts @@ -432,3 +432,50 @@ describe('v0.41.2.1: runPhaseExtractAtoms — dual-source merge + idempotency', expect(discovered.details?.atoms_extracted).toBe(1); }); }); + +describe('#2144: zero-yield tombstone', () => { + test('zero-yield page is stamped and excluded from rediscovery', async () => { + await seedPage({ slug: 'article/zero-yield', type: 'article' }); + // Successful LLM call that yields no atoms. + const result = await runPhaseExtractAtoms(engine, { _transcripts: [], _chat: stubChat('[]') }); + expect(result.details?.pages_processed).toBe(1); + expect(result.details?.atoms_extracted).toBe(0); + + // Stamp landed: atoms_scan_hash = first 16 chars of the page's content_hash. + const rows = await engine.executeRaw<{ scan: string; ch: string }>( + `SELECT frontmatter->>'atoms_scan_hash' AS scan, content_hash AS ch + FROM pages WHERE slug = 'article/zero-yield'`, + ); + expect(rows[0].scan).toBe(rows[0].ch.slice(0, 16)); + + // No longer rediscovered. + const discovered = await discoverExtractablePages(engine, 'default'); + expect(discovered.find((d) => d.slug === 'article/zero-yield')).toBeUndefined(); + }); + + test('content change re-eligibilizes a tombstoned page', async () => { + await seedPage({ slug: 'article/evolves', type: 'article' }); + await runPhaseExtractAtoms(engine, { _transcripts: [], _chat: stubChat('[]') }); + expect((await discoverExtractablePages(engine, 'default')).length).toBe(0); + + // Simulate an edit: content_hash moves while the stale stamp stays. + await engine.executeRaw( + `UPDATE pages SET content_hash = 'fresh-hash-after-edit' WHERE slug = $1 AND source_id = 'default'`, + ['article/evolves'], + ); + const rediscovered = await discoverExtractablePages(engine, 'default'); + expect(rediscovered.map((d) => d.slug)).toContain('article/evolves'); + }); + + test('failed chat does NOT stamp — page stays retryable', async () => { + await seedPage({ slug: 'article/transient-failure', type: 'article' }); + const failingChat = async (_o: ChatOpts): Promise => { throw new Error('rate limit'); }; + await runPhaseExtractAtoms(engine, { _transcripts: [], _chat: failingChat as never }); + const rows = await engine.executeRaw<{ scan: string | null }>( + `SELECT frontmatter->>'atoms_scan_hash' AS scan FROM pages WHERE slug = 'article/transient-failure'`, + ); + expect(rows[0].scan).toBeNull(); + const discovered = await discoverExtractablePages(engine, 'default'); + expect(discovered.map((d) => d.slug)).toContain('article/transient-failure'); + }); +}); diff --git a/test/extract-stale.test.ts b/test/extract-stale.test.ts index a725e309b..f6db5ff18 100644 --- a/test/extract-stale.test.ts +++ b/test/extract-stale.test.ts @@ -209,6 +209,32 @@ describe('gbrain extract --stale', () => { expect(usRows[0]?.eq).toBe(true); }); + test('REGRESSION: page with updated_at BEFORE LINK_EXTRACTOR_VERSION_TS clears (no permanent-stale loop)', async () => { + // The v112 watermark column ships with no backfill, so every pre-existing + // page starts NULL-stale — and most pre-date the version bump. Pre-fix, + // extractStaleFromDB stamped links_extracted_at = read updated_at; for a + // page edited before LINK_EXTRACTOR_VERSION_TS the stamp landed BELOW the + // version threshold, so the version arm (links_extracted_at < versionTs) + // re-flagged it stale forever — an infinite re-extract loop that never + // cleared the lag (observed: 97% of pages stuck permanently). + await engine.putPage('people/alice', personPage('Alice')); + await engine.putPage('companies/acme', companyPage('Acme', '[Alice](people/alice) leads [Acme](companies/acme).')); + // Backdate every page to BEFORE the extractor version timestamp. + await engine.executeRaw(`UPDATE pages SET updated_at = '2020-01-01T00:00:00Z'`); + expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(2); + + await runExtract(engine, ['--stale']); + // Fixed: stamp = GREATEST(read updated_at, versionTs) → lifts old pages to + // the threshold so the version arm clears, while a real future edit still + // advances updated_at past the stamp (CDX-1 race protection preserved). + expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(0); + + // Second run must ALSO find 0 — the defining symptom of the bug was that it + // never converged. + await runExtract(engine, ['--stale']); + expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(0); + }); + test('CDX-4 (D2): a link-flush throw aborts the sweep and leaves pages UNSTAMPED', async () => { await engine.putPage('people/alice', personPage('Alice')); await engine.putPage('companies/acme', companyPage('Acme', '[Alice](people/alice) founded [Acme](companies/acme).')); diff --git a/test/link-extraction.test.ts b/test/link-extraction.test.ts index f7c003ab5..64e1db2f9 100644 --- a/test/link-extraction.test.ts +++ b/test/link-extraction.test.ts @@ -140,6 +140,17 @@ describe('extractEntityRefs', () => { expect(wikiRefs[0].needsResolution).toBe(true); }); + test('recognizes reference-page wikilinks as concrete targets', () => { + const refs = extractEntityRefs('See [[reference/mcminnville-market-data]] for source context.'); + expect(refs.length).toBe(1); + expect(refs[0]).toMatchObject({ + name: 'reference/mcminnville-market-data', + slug: 'reference/mcminnville-market-data', + dir: 'reference', + }); + expect(refs[0].needsResolution).toBeUndefined(); + }); + test('skips qualified-syntax tokens (those belong to 2a)', () => { // [[wiki:topics/ai]] looks like 2a's qualified shape — even though // it wouldn't satisfy DIR_PATTERN, 2c must not claim it either @@ -1236,6 +1247,43 @@ describe('makeResolver — fallback chain', () => { const out = await r.resolveBasenameMatches!('struktura'); expect(out.sort()).toEqual(['notes/struktura', 'struktura']); }); + + test('opts.sourceId is forwarded to findByTitleFuzzy (twin of #1436 fix)', async () => { + // Captures every (name, dirPrefix, minSimilarity, sourceId) call so we + // can assert the resolver threads sourceId through. Without the wire-up, + // findByTitleFuzzy would be called with sourceId=undefined and the SQL + // could return cross-source slug suggestions that the FK filter + // downstream silently drops. + const calls: Array<{ name: string; dirPrefix?: string; minSimilarity?: number; sourceId?: string }> = []; + const engine = { + async getPage() { return null; }, + async findByTitleFuzzy(name: string, dirPrefix?: string, minSimilarity?: number, sourceId?: string) { + calls.push({ name, dirPrefix, minSimilarity, sourceId }); + return null; + }, + async searchKeyword() { return []; }, + } as unknown as BrainEngine; + const r = makeResolver(engine, { mode: 'batch', sourceId: 'src-a' }); + await r.resolve('Alice Example', 'people'); + expect(calls.length).toBeGreaterThan(0); + expect(calls.every(c => c.sourceId === 'src-a')).toBe(true); + }); + + test('opts.sourceId omitted → findByTitleFuzzy receives undefined (back-compat)', async () => { + const calls: Array<{ sourceId?: string }> = []; + const engine = { + async getPage() { return null; }, + async findByTitleFuzzy(_name: string, _dirPrefix?: string, _min?: number, sourceId?: string) { + calls.push({ sourceId }); + return null; + }, + async searchKeyword() { return []; }, + } as unknown as BrainEngine; + const r = makeResolver(engine, { mode: 'batch' }); + await r.resolve('Alice Example', 'people'); + expect(calls.length).toBeGreaterThan(0); + expect(calls.every(c => c.sourceId === undefined)).toBe(true); + }); }); describe('FRONTMATTER_LINK_MAP integrity', () => { diff --git a/test/lint.test.ts b/test/lint.test.ts index ca38b8d23..f843d5859 100644 --- a/test/lint.test.ts +++ b/test/lint.test.ts @@ -32,6 +32,29 @@ describe('lintContent', () => { expect(issues.some(i => i.rule === 'code-fence-wrap')).toBe(true); }); + test('no false positive: page CONTAINS an inner ```markdown code block', () => { + // Real-world case: a docs/SKILL page that shows a markdown example inline. + // Before this fix, the detector used the /m flag so ^/$ matched start/end + // of any line, which fired on any file that simply contained a ```markdown + // line. But fixContent's regex has no /m flag and can only strip whole-file + // wrappers, so the issue was reported as "fixable: true" yet never fixed. + const content = + '---\ntitle: Skill\n---\n\n# Skill\n\nExample input shape:\n\n' + + '```markdown\n# Inner page\nContent.\n```\n\nThat ends the example.\n'; + const issues = lintContent(content, 'test.md'); + expect(issues.filter(i => i.rule === 'code-fence-wrap')).toHaveLength(0); + }); + + test('no false positive: multiple inner ```markdown blocks', () => { + // Documentation pages frequently include several markdown examples. + const content = + '---\ntitle: Examples\n---\n\n# Examples\n\nFirst:\n\n' + + '```markdown\nfoo\n```\n\nSecond:\n\n' + + '```markdown\nbar\n```\n\nDone.\n'; + const issues = lintContent(content, 'test.md'); + expect(issues.filter(i => i.rule === 'code-fence-wrap')).toHaveLength(0); + }); + test('detects placeholder dates', () => { const content = '---\ntitle: Test\ntype: person\ncreated: YYYY-MM-DD\n---\n\n# Test'; const issues = lintContent(content, 'test.md'); diff --git a/test/loadConfig-merge.test.ts b/test/loadConfig-merge.test.ts index 7a8ab08af..e5a861960 100644 --- a/test/loadConfig-merge.test.ts +++ b/test/loadConfig-merge.test.ts @@ -9,6 +9,7 @@ import { loadConfigWithEngine, type GBrainConfig } from '../src/core/config.ts'; interface FakeEngine { getConfig(key: string): Promise; + listConfigKeys?(prefix: string): Promise; } function makeEngine(map: Record): FakeEngine { @@ -16,6 +17,9 @@ function makeEngine(map: Record): FakeEngine async getConfig(key: string) { return map[key]; }, + async listConfigKeys(prefix: string) { + return Object.keys(map).filter(key => key.startsWith(prefix)); + }, }; } @@ -92,6 +96,31 @@ describe('loadConfigWithEngine (Phase 4 / F3)', () => { expect(merged?.embedding_image_ocr).toBe(true); }); + test('DB provider_base_urls. fills the gateway base URL map', async () => { + const base: GBrainConfig = { engine: 'pglite' }; + const engine = makeEngine({ + 'provider_base_urls.llama-server-reranker': 'http://127.0.0.1:8091/v1', + }); + const merged = await loadConfigWithEngine(engine, base); + expect(merged?.provider_base_urls?.['llama-server-reranker']).toBe('http://127.0.0.1:8091/v1'); + }); + + test('provider_base_urls merge is per-provider: file value wins and DB fills siblings', async () => { + const base: GBrainConfig = { + engine: 'pglite', + provider_base_urls: { + 'llama-server-reranker': 'http://file.example/v1', + }, + }; + const engine = makeEngine({ + 'provider_base_urls.llama-server-reranker': 'http://db.example/v1', + 'provider_base_urls.openrouter': 'http://openrouter.example/v1', + }); + const merged = await loadConfigWithEngine(engine, base); + expect(merged?.provider_base_urls?.['llama-server-reranker']).toBe('http://file.example/v1'); + expect(merged?.provider_base_urls?.openrouter).toBe('http://openrouter.example/v1'); + }); + test('engine.getConfig throwing is non-fatal — file/env config still returned', async () => { const base: GBrainConfig = { engine: 'pglite', diff --git a/test/openrouter-reranker-recipe.test.ts b/test/openrouter-reranker-recipe.test.ts new file mode 100644 index 000000000..508b7b945 --- /dev/null +++ b/test/openrouter-reranker-recipe.test.ts @@ -0,0 +1,44 @@ +import { describe, test, expect } from 'bun:test'; +import { getRecipe } from '../src/core/ai/recipes/index.ts'; + +describe('OpenRouter recipe — reranker touchpoint', () => { + test('declares a reranker touchpoint', () => { + const r = getRecipe('openrouter'); + expect(r).toBeDefined(); + expect(r!.touchpoints.reranker).toBeDefined(); + }); + + test('models list includes all supported IDs (incl. NVIDIA :free suffix)', () => { + const m = getRecipe('openrouter')!.touchpoints.reranker!.models; + expect(m).toContain('cohere/rerank-v3.5'); + expect(m).toContain('cohere/rerank-4-fast'); + expect(m).toContain('cohere/rerank-4-pro'); + // The :free suffix must appear in full — gateway.rerank() does exact + // string matching against the allowlist (no v0.31.12 extended-model bypass + // on the rerank path), so truncating to `nvidia/.../v2` would 403. + expect(m).toContain('nvidia/llama-nemotron-rerank-vl-1b-v2:free'); + }); + + test('default_model is cohere/rerank-v3.5', () => { + const tp = getRecipe('openrouter')!.touchpoints.reranker!; + expect(tp.default_model).toBe('cohere/rerank-v3.5'); + expect(tp.models).toContain(tp.default_model); + }); + + test('path is /rerank (NOT ZeroEntropy default /models/rerank)', () => { + const tp = getRecipe('openrouter')!.touchpoints.reranker!; + expect(tp.path).toBe('/rerank'); + }); + + test('max_payload_bytes and timeout match plan', () => { + const tp = getRecipe('openrouter')!.touchpoints.reranker!; + expect(tp.max_payload_bytes).toBe(5_000_000); + expect(tp.default_timeout_ms).toBe(5_000); + }); + + test('cost_per_1m_tokens_usd is set (pseudo-rate for per-search billing)', () => { + const tp = getRecipe('openrouter')!.touchpoints.reranker!; + expect(typeof tp.cost_per_1m_tokens_usd).toBe('number'); + expect(tp.cost_per_1m_tokens_usd).toBeGreaterThan(0); + }); +}); diff --git a/test/progress.test.ts b/test/progress.test.ts index 0e70e0d6f..7f266d770 100644 --- a/test/progress.test.ts +++ b/test/progress.test.ts @@ -218,15 +218,21 @@ describe('progress reporter', () => { test('only one process-level signal handler installed across many reporters', () => { // Baseline: one handler already installed by prior tests in this file. const installedBefore = __signalHandlerInstalledForTest(); + // liveReporters is module-global, so a reporter left running by ANOTHER + // test file in the same shard shows up here. Assert the DELTA (these 50 + // lifecycles leak nothing) instead of an absolute zero — the absolute + // form flaked whenever shard composition changed and an unrelated file + // held a live reporter across this test. + const liveBefore = __liveReporterCountForTest(); const { stream } = sink(false); for (let i = 0; i < 50; i++) { const p = createProgress({ mode: 'json', stream, minIntervalMs: 0, minItems: 1 }); p.start(`phase_${i}`, 1); p.finish(); } - // After 50 reporter lifecycles, still exactly one handler and zero leaked live entries. + // After 50 reporter lifecycles, still exactly one handler and no new live entries. expect(__signalHandlerInstalledForTest()).toBe(installedBefore || true); - expect(__liveReporterCountForTest()).toBe(0); + expect(__liveReporterCountForTest()).toBe(liveBefore); }); test('startHeartbeat() fires heartbeats and stop() clears', async () => { diff --git a/test/think-gaps.test.ts b/test/think-gaps.test.ts new file mode 100644 index 000000000..3b1660700 --- /dev/null +++ b/test/think-gaps.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, test } from 'bun:test'; +import { stripGapsSection } from '../src/core/think/index.ts'; +import { buildThinkSystemPrompt } from '../src/core/think/prompt.ts'; + +// `gbrain think` returns gaps in the structured `gaps` array, which both the +// CLI (`src/commands/think.ts`) and the persisted synthesis page +// (`persistSynthesis`) render exactly once. Older prompts also asked for a +// "Gaps" section inside the answer prose, so a model that still emits one made +// the output print "## Gaps" twice. `stripGapsSection` removes the prose +// section so the structured array is the single source of truth. + +describe('stripGapsSection', () => { + test('removes a trailing "## Gaps" section', () => { + const answer = 'The answer with a claim [people/alice].\n\n## Gaps\n- no update since 2026-03-22 [projects/acme]\n- pricing not recorded'; + const out = stripGapsSection(answer); + expect(out).not.toContain('## Gaps'); + expect(out).not.toContain('no update since'); + expect(out).toContain('The answer with a claim [people/alice].'); + }); + + test('removes a level-3 "### Gaps" section', () => { + const out = stripGapsSection('Body text.\n\n### Gaps\n- missing thing'); + expect(out).not.toMatch(/#+\s+Gaps/i); + expect(out).toBe('Body text.'); + }); + + test('is case-insensitive', () => { + expect(stripGapsSection('Body.\n\n## GAPS\n- x')).toBe('Body.'); + expect(stripGapsSection('Body.\n\n## gaps\n- x')).toBe('Body.'); + }); + + test('returns the answer unchanged when there is no Gaps section', () => { + const answer = 'Just an answer.\n\n## Conflicts\n- a vs b'; + expect(stripGapsSection(answer)).toBe(answer); + }); + + test('does not match a heading that merely starts with "Gaps"', () => { + const answer = 'Body.\n\n## Gaps in the coverage\n- this is real content'; + expect(stripGapsSection(answer)).toBe(answer); + }); + + test('stops at the next same-or-higher heading (preserves later content)', () => { + const answer = 'Intro.\n\n## Gaps\n- missing x\n\n## Sources\n- [a]'; + const out = stripGapsSection(answer); + expect(out).not.toContain('missing x'); + expect(out).toContain('## Sources'); + expect(out).toContain('- [a]'); + }); + + test('handles empty / falsy input', () => { + expect(stripGapsSection('')).toBe(''); + }); + + test('the bug repro: strip + structured render yields exactly one "## Gaps"', () => { + // Mirrors the render in src/commands/think.ts: print the (stripped) answer, + // then append one "## Gaps" block from the structured `gaps` array. + const answer = 'Answer prose [people/alice].\n\n## Gaps\n- the prose gap, slightly different wording'; + const gaps = ['the structured gap']; + const rendered = + stripGapsSection(answer) + '\n\n## Gaps\n' + gaps.map((g) => `- ${g}`).join('\n'); + expect((rendered.match(/## Gaps/g) ?? []).length).toBe(1); + expect(rendered).toContain('the structured gap'); + }); +}); + +describe('buildThinkSystemPrompt — gaps go in the structured array, not the answer body', () => { + test('the answer schema no longer lists "Gaps" as a body section', () => { + const out = buildThinkSystemPrompt({}); + expect(out).not.toContain('Sections: Answer, Conflicts (optional), Gaps'); + expect(out).toContain('gaps belong in the gaps array'); + }); + + test('still requires the structured "gaps" array', () => { + const out = buildThinkSystemPrompt({}); + expect(out).toContain('"gaps"'); + }); + + test('preserves the Conflicts section and the Hard rules', () => { + const out = buildThinkSystemPrompt({}); + expect(out).toContain('Conflicts'); + expect(out).toContain('Hard rules:'); + expect(out).toContain('Cite EVERY substantive claim'); + }); + + test('willSave mode routes gaps to the structured array (no body Gaps section)', () => { + const out = buildThinkSystemPrompt({ willSave: true }); + expect(out).not.toContain('cover Answer, Conflicts, and Gaps thoroughly'); + expect(out).toContain('structured "gaps" array'); + }); +});