mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
Merge remote-tracking branch 'origin/master' into pr3186-work
This commit is contained in:
Garry Tan
@@ -206,7 +206,11 @@ jobs:
|
||||
needs: cache-check
|
||||
if: needs.cache-check.outputs.hit != 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
# 22, not 15: under parallel PR load the PGLite WASM cold-starts stretch a
|
||||
# shard past 15 min while every test is still passing — the timeout then
|
||||
# cancels the job and the test-status gate reads it as a failure. 13 runs
|
||||
# died this way on 2026-07-21/22 alone.
|
||||
timeout-minutes: 22
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
|
||||
@@ -1186,6 +1186,15 @@ function writeWrapperScript(repoPath: string): string {
|
||||
# OPENAI/ANTHROPIC keys exported in zshenv reach autopilot.
|
||||
[ -f ~/.zshenv ] && source ~/.zshenv 2>/dev/null
|
||||
source ~/.zshrc 2>/dev/null || source ~/.bashrc 2>/dev/null || true
|
||||
# Belt-and-suspenders PATH fix. ~/.bashrc ships with a non-interactive guard
|
||||
# (\`case $- in *i*) ;; *) return;; esac\`) that exits early when launched from
|
||||
# cron/systemd/launchd — so its PATH exports never reach this subprocess.
|
||||
# Without bun on PATH, the exec'd gbrain (a \`#!/usr/bin/env bun\` script) fails
|
||||
# silently with "env: bun: No such file or directory" and leaves a stale
|
||||
# lockfile that blocks every subsequent tick. Prepending ~/.bun/bin here
|
||||
# keeps the wrapper self-contained regardless of which init file the OS
|
||||
# loaded.
|
||||
export PATH="$HOME/.bun/bin:$PATH"
|
||||
exec '${safeGbrainPath}' autopilot --repo '${safeRepoPath}'
|
||||
`;
|
||||
writeFileSync(wrapperPath, wrapper, { mode: 0o755 });
|
||||
|
||||
@@ -37,9 +37,19 @@ export async function findCodeDef(
|
||||
// trigger) are first-class definitions in the SQL sense. The chunker's
|
||||
// normalizeSymbolType maps create_table → 'table' etc, so adding the SQL
|
||||
// kinds here is what makes `gbrain code-def users` work against SQL.
|
||||
// Method-level + member definitions. normalizeSymbolType only canonicalizes
|
||||
// some node types; the rest fall through `type.replace(/_/g, ' ')`, so
|
||||
// tree-sitter's method_declaration → 'method declaration', struct_specifier →
|
||||
// 'struct specifier', protocol_declaration → 'protocol declaration', etc.
|
||||
// Without these, code-def is blind to every method, constructor, field, C
|
||||
// struct, and Swift protocol — which is most of an OO codebase. The plain
|
||||
// 'struct' entry above never matched for the same reason (C emits the
|
||||
// 'struct specifier' fallback form).
|
||||
const DEF_TYPES = [
|
||||
'function', 'class', 'interface', 'type', 'enum', 'struct', 'trait', 'module', 'contract',
|
||||
'table', 'view', 'index', 'procedure', 'schema', 'database', 'trigger',
|
||||
'method declaration', 'method definition', 'constructor declaration',
|
||||
'field declaration', 'field definition', 'struct specifier', 'protocol declaration',
|
||||
];
|
||||
const params: unknown[] = [symbol, limit];
|
||||
let whereLang = '';
|
||||
|
||||
+53
-10
@@ -4963,8 +4963,7 @@ export async function buildChecks(
|
||||
message:
|
||||
`${unmatched}/${sample.length} conversation pages (${unmatchedPct.toFixed(1)}%) match NO built-in pattern. ` +
|
||||
`Breakdown: ${breakdown}. ` +
|
||||
`Investigate: gbrain conversation-parser scan <slug> | ` +
|
||||
`Enable LLM fallback (opt-in): gbrain config set conversation_parser.llm_fallback_enabled true`,
|
||||
`Investigate: gbrain conversation-parser scan <slug>`,
|
||||
});
|
||||
} else {
|
||||
checks.push({
|
||||
@@ -7778,27 +7777,71 @@ export async function runRemediationPlan(
|
||||
return;
|
||||
}
|
||||
|
||||
// Human output
|
||||
console.log(`Brain score: ${plan.brain_score_current}/100 → target ${targetScore}`);
|
||||
for (const line of renderRemediationPlanLines(plan, targetScore)) {
|
||||
console.log(line);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Human-render the remediation plan into a sequence of console lines.
|
||||
* Exported for unit-test access — `runRemediationPlan` consumes it
|
||||
* verbatim and only adds the JSON-mode short-circuit.
|
||||
*
|
||||
* Gating the "at target" line on `brain_score_current >= targetScore`
|
||||
* is load-bearing: when the plan is empty AND the target is unreachable,
|
||||
* the prior shape printed both "Target unreachable: …" and "Brain is at
|
||||
* target" back-to-back, which contradicted itself and hid the real next
|
||||
* step (manual prereq config to lift `max_reachable_score`).
|
||||
*/
|
||||
export function renderRemediationPlanLines(
|
||||
plan: RemediationPlanShape,
|
||||
targetScore: number,
|
||||
): string[] {
|
||||
const lines: string[] = [];
|
||||
lines.push(`Brain score: ${plan.brain_score_current}/100 → target ${targetScore}`);
|
||||
if (plan.target_unreachable) {
|
||||
console.log(`Target unreachable: max with autonomous remediation is ${plan.max_reachable_score}/100.`);
|
||||
lines.push(`Target unreachable: max with autonomous remediation is ${plan.max_reachable_score}/100.`);
|
||||
}
|
||||
if (plan.plan.length === 0) {
|
||||
console.log('No remediations needed. Brain is at target.');
|
||||
if (plan.brain_score_current >= targetScore) {
|
||||
lines.push('No remediations needed. Brain is at target.');
|
||||
}
|
||||
// When brain_score < targetScore and plan is empty, the unreachable
|
||||
// line (if applicable) is the user-facing explanation; the blocked-
|
||||
// checks block below surfaces the manual gap. Don't follow with a
|
||||
// misleading "at target" claim.
|
||||
} else {
|
||||
console.log(`Plan: ${plan.plan.length} step(s), est ${plan.est_total_seconds}s, est $${plan.est_total_usd_cost.toFixed(2)}`);
|
||||
lines.push(`Plan: ${plan.plan.length} step(s), est ${plan.est_total_seconds}s, est $${plan.est_total_usd_cost.toFixed(2)}`);
|
||||
for (const step of plan.plan) {
|
||||
const protectedMark = step.protected ? ' [PROTECTED]' : '';
|
||||
const costMark = step.est_usd_cost ? ` ($${step.est_usd_cost.toFixed(2)})` : '';
|
||||
console.log(` ${step.step}. [${step.severity}] ${step.job}${protectedMark} — ${step.rationale}${costMark}`);
|
||||
lines.push(` ${step.step}. [${step.severity}] ${step.job}${protectedMark} — ${step.rationale}${costMark}`);
|
||||
}
|
||||
}
|
||||
if (plan.blocked.length > 0) {
|
||||
console.log(`\nBlocked checks (prereq missing):`);
|
||||
lines.push(`\nBlocked checks (prereq missing):`);
|
||||
for (const b of plan.blocked) {
|
||||
console.log(` - ${b.check}: ${b.reason}`);
|
||||
lines.push(` - ${b.check}: ${b.reason}`);
|
||||
}
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
interface RemediationPlanShape {
|
||||
brain_score_current: number;
|
||||
target_unreachable: boolean;
|
||||
max_reachable_score: number;
|
||||
plan: Array<{
|
||||
step: number;
|
||||
severity: string;
|
||||
job: string;
|
||||
protected?: boolean;
|
||||
est_usd_cost?: number;
|
||||
rationale: string;
|
||||
}>;
|
||||
est_total_seconds: number;
|
||||
est_total_usd_cost: number;
|
||||
blocked: Array<{ check: string; reason: string }>;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+12
-1
@@ -1743,7 +1743,18 @@ export async function extractStaleFromDB(
|
||||
// `page.updated_at.toISOString()` — the JS Date is ms-truncated, so the
|
||||
// µs-precision DB updated_at stayed strictly greater and the page never
|
||||
// cleared on Postgres. Stamping the exact value makes them equal.
|
||||
processedRefs.push({ slug: page.slug, source_id: page.source_id, extractedAt: page.updated_at_iso });
|
||||
//
|
||||
// BUT the stamp must also clear the version-staleness clause
|
||||
// (`links_extracted_at < versionTs`). A page whose updated_at predates
|
||||
// versionTs would otherwise be stamped below the threshold and read as
|
||||
// stale forever — a permanent re-extract loop that never clears the lag.
|
||||
// GREATEST(updated_at, versionTs) preserves the race semantics (a real
|
||||
// future edit advances updated_at > versionTs >= stamp → re-extracts)
|
||||
// while lifting old pages to the threshold so they clear.
|
||||
const stampIso = page.updated_at.getTime() >= Date.parse(versionTs)
|
||||
? page.updated_at_iso
|
||||
: versionTs;
|
||||
processedRefs.push({ slug: page.slug, source_id: page.source_id, extractedAt: stampIso });
|
||||
}
|
||||
|
||||
// Flush NON-swallowing (CDX-4): a throw here propagates out of the sweep so
|
||||
|
||||
@@ -98,8 +98,17 @@ export function findBareTweetHits(compiledTruth: string, slug: string): BareTwee
|
||||
}
|
||||
// If the line already contains a tweet URL, it's cited — skip
|
||||
if (URL_NEARBY_RE.test(line)) continue;
|
||||
// If the line carries an explicit source citation (e.g.
|
||||
// "[Source: X, @handle, 2026-05-28]"), it's already attributed — skip.
|
||||
// Catches instructional/example lines in recipe docs that demonstrate
|
||||
// the CORRECT citation format. (v0.42.x)
|
||||
if (/\[\s*source:/i.test(line)) continue;
|
||||
// Strip inline-code spans (`...`) before matching: phrases shown as
|
||||
// inline-code templates in docs are examples, not bare claims. The
|
||||
// fenced-code skip above only covers ``` blocks, not inline backticks.
|
||||
const lineForMatch = line.replace(/`[^`]*`/g, '');
|
||||
for (const re of BARE_TWEET_PHRASES) {
|
||||
const m = line.match(re);
|
||||
const m = lineForMatch.match(re);
|
||||
if (m) {
|
||||
hits.push({ slug, line: i + 1, rawLine: line.trim(), phrase: m[0] });
|
||||
break; // one finding per line is enough
|
||||
|
||||
@@ -1664,7 +1664,13 @@ export async function registerBuiltinHandlers(
|
||||
|
||||
worker.register('backlinks', async (job) => {
|
||||
const { runBacklinksCore } = await import('./backlinks.ts');
|
||||
const action: 'check' | 'fix' = job.data.action === 'check' ? 'check' : 'fix';
|
||||
// Default to 'check', not 'fix': backlinks jobs submitted with an empty
|
||||
// payload (e.g. the sync→embed→backlinks chains enqueued after ingestion)
|
||||
// must never rewrite tracked brain pages with generated "Referenced in"
|
||||
// timeline bullets. Mirrors the documented intent in src/core/cycle.ts
|
||||
// (runPhaseBacklinks). The filesystem fixer stays available explicitly
|
||||
// via '{"action":"fix"}' or `gbrain check-backlinks fix`.
|
||||
const action: 'check' | 'fix' = job.data.action === 'fix' ? 'fix' : 'check';
|
||||
const dir = typeof job.data.dir === 'string'
|
||||
? job.data.dir
|
||||
: (await engine.getConfig('sync.repo_path')) ?? '.';
|
||||
|
||||
@@ -127,7 +127,12 @@ export function lintContent(content: string, filePath: string, opts: LintContent
|
||||
}
|
||||
|
||||
// Rule: Wrapping code fences (```markdown ... ```)
|
||||
if (content.match(/^```(?:markdown|md)\s*\n/m) && content.match(/\n```\s*$/m)) {
|
||||
// Detector intentionally has NO /m flag so ^/$ match start/end of the whole
|
||||
// file, not inner lines. Keeps detector in sync with fixContent() below,
|
||||
// which also has no /m flag. Without this, lint reports "fixable" false
|
||||
// positives on any page that simply contains a ```markdown code block, but
|
||||
// fixContent can never strip them (its regex only matches whole-file wrappers).
|
||||
if (content.match(/^```(?:markdown|md)\s*\n/) && content.match(/\n```\s*$/)) {
|
||||
issues.push({
|
||||
file: filePath, line: 1, rule: 'code-fence-wrap',
|
||||
message: 'Page wrapped in ```markdown code fences (LLM artifact)',
|
||||
|
||||
+14
-1
@@ -536,7 +536,20 @@ function shouldSkipProvider(modelStr: string, skip: string[]): boolean {
|
||||
|
||||
export async function runModels(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
const json = args.includes('--json');
|
||||
const sub = args[1] === 'doctor' ? 'doctor' : args[1] === 'help' || args.includes('--help') || args.includes('-h') ? 'help' : 'read';
|
||||
// args is `subArgs` from cli.ts `handleCliOnly` — the leading 'models'
|
||||
// token has already been stripped. The subcommand is at args[0], NOT
|
||||
// args[1]. Pre-fix this check was `args[1]`, so `gbrain models doctor`
|
||||
// silently fell through to the read view. The doctor probe path was
|
||||
// unreachable from the CLI.
|
||||
//
|
||||
// --help honored FIRST so `gbrain models doctor --help` shows usage
|
||||
// instead of running network probes (which would spend tokens or
|
||||
// exit nonzero when the user only asked for help). Pre-fix the
|
||||
// args[1] ternary happened to dodge this by always falling through
|
||||
// to the args.includes('--help') branch; the args[0] rewrite needs
|
||||
// explicit ordering to preserve that behavior.
|
||||
const hasHelp = args.includes('--help') || args.includes('-h') || args[0] === 'help';
|
||||
const sub = hasHelp ? 'help' : args[0] === 'doctor' ? 'doctor' : 'read';
|
||||
|
||||
if (sub === 'help') {
|
||||
process.stdout.write(
|
||||
|
||||
@@ -843,6 +843,21 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
// reverse proxies / tunnels; default to localhost for dev.
|
||||
const issuerUrl = new URL(publicUrl || `http://localhost:${port}`);
|
||||
|
||||
// MCP authorization spec (2025-06-18 draft §5.1) and RFC 9728 require the
|
||||
// protected resource server to return its discovery metadata URL in the
|
||||
// WWW-Authenticate header on 401 responses:
|
||||
//
|
||||
// WWW-Authenticate: Bearer resource_metadata="<URL>"
|
||||
//
|
||||
// Clients (claude.ai, Cursor, every other MCP-aware OAuth client) use that
|
||||
// URL to find the authorization-server discovery doc + token endpoint
|
||||
// without the user having to paste those URLs manually. Pre-fix the header
|
||||
// shipped `Bearer error="invalid_token", ...` with no resource_metadata
|
||||
// parameter, so MCP clients couldn't begin the OAuth flow from a fresh
|
||||
// 401 — they would silently fail to connect with a generic "couldn't
|
||||
// reach the MCP server" error.
|
||||
const resourceMetadataUrl = `${issuerUrl.toString().replace(/\/$/, '')}/.well-known/oauth-protected-resource`;
|
||||
|
||||
// F9: cookie `secure` flag honors both the request's TLS state (req.secure
|
||||
// is set when express trust-proxy lands an X-Forwarded-Proto: https) AND
|
||||
// the operator's declared issuer protocol (so a Cloudflare-tunnel deploy
|
||||
@@ -1601,7 +1616,7 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
res.status(405).json({ jsonrpc: '2.0', error: { code: -32000, message: 'Method not allowed' }, id: null });
|
||||
});
|
||||
|
||||
app.post('/mcp', requireBearerAuth({ verifier: oauthProvider }), async (req: Request, res: Response) => {
|
||||
app.post('/mcp', requireBearerAuth({ verifier: oauthProvider, resourceMetadataUrl }), async (req: Request, res: Response) => {
|
||||
const startTime = Date.now();
|
||||
const authInfo = (req as any).auth as AuthInfo;
|
||||
|
||||
@@ -1944,7 +1959,7 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
app.post(
|
||||
'/ingest',
|
||||
ingestRateLimiter,
|
||||
requireBearerAuth({ verifier: oauthProvider, requiredScopes: ['write'] }),
|
||||
requireBearerAuth({ verifier: oauthProvider, requiredScopes: ['write'], resourceMetadataUrl }),
|
||||
express.raw({ type: '*/*', limit: ingestMaxBytes }),
|
||||
async (req: Request, res: Response) => {
|
||||
const startTime = Date.now();
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
* degrades to gather-only output with a warning if missing.
|
||||
*/
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import { runThink, persistSynthesis } from '../core/think/index.ts';
|
||||
import { runThink, persistSynthesis, stripGapsSection } from '../core/think/index.ts';
|
||||
import { loadConfig, isThinClient } from '../core/config.ts';
|
||||
import { callRemoteTool, unpackToolResult } from '../core/mcp-client.ts';
|
||||
|
||||
@@ -157,7 +157,7 @@ prints what would have been the input (exit 0).
|
||||
|
||||
// Human-readable output
|
||||
console.log(`# ${question}\n`);
|
||||
console.log(result.answer);
|
||||
console.log(stripGapsSection(result.answer));
|
||||
console.log('');
|
||||
if (result.gaps.length > 0) {
|
||||
console.log('## Gaps');
|
||||
|
||||
@@ -263,6 +263,15 @@ export function dimsProviderOptions(
|
||||
if (modelId === 'text-embedding-v3' || modelId === 'embedding-3') {
|
||||
return { openaiCompatible: { dimensions: dims } };
|
||||
}
|
||||
// Qwen3-Embedding family on Ollama (and any other openai-compatible
|
||||
// provider serving it) supports Matryoshka truncation via `dimensions`.
|
||||
// Native sizes: 0.6B=1024, 4B=2560, 8B=4096. Without `dimensions`,
|
||||
// Ollama returns the native size and brains configured for narrower
|
||||
// widths hard-fail with a dim-mismatch error. Pattern match the bare
|
||||
// model name + any `:tag` (e.g. `qwen3-embedding:4b`, `qwen3-embedding:0.6b`).
|
||||
if (modelId === 'qwen3-embedding' || modelId.startsWith('qwen3-embedding:')) {
|
||||
return { openaiCompatible: { dimensions: dims } };
|
||||
}
|
||||
// MiniMax embo-01 takes a `type: 'db' | 'query'` field for asymmetric
|
||||
// retrieval. Today still hardcoded to 'db' for back-compat — opting
|
||||
// into the new inputType seam is a follow-up (see plan's deferred
|
||||
|
||||
+27
-1
@@ -599,6 +599,8 @@ function warnRecipesMissingBatchTokens(): void {
|
||||
// LiteLLM proxy, llama-server) — they ship without a static cap because
|
||||
// the cap depends on a user-launched server. Warning is noise for them.
|
||||
if (embedding.no_batch_cap === true) continue;
|
||||
// A declared item-count cap is a real batch cap — no warning needed.
|
||||
if (embedding.max_batch_items !== undefined) continue;
|
||||
if (_warnedRecipes.has(recipe.id)) continue;
|
||||
_warnedRecipes.add(recipe.id);
|
||||
// eslint-disable-next-line no-console
|
||||
@@ -1517,10 +1519,17 @@ export async function embed(texts: string[], opts?: EmbedOpts): Promise<Float32A
|
||||
|
||||
// Pre-split is gated on max_batch_tokens. Recipes without it (e.g. OpenAI)
|
||||
// ride the fast path: one embedMany call, no recursion safety net.
|
||||
const batches = maxBatchTokens
|
||||
const tokenBatches = maxBatchTokens
|
||||
? splitByTokenBudget(truncated, Math.floor(maxBatchTokens * effectiveSafetyFactor(recipe)), charsPerToken)
|
||||
: [truncated];
|
||||
|
||||
// Hard COUNT cap (e.g. llama-server's "maximum allowed batch size 32").
|
||||
// Token budget can't bound item count, so re-split any oversized batch.
|
||||
const maxBatchItems = embedding?.max_batch_items;
|
||||
const batches = maxBatchItems
|
||||
? tokenBatches.flatMap(b => capBatchItems(b, maxBatchItems))
|
||||
: tokenBatches;
|
||||
|
||||
const allEmbeddings: Float32Array[] = [];
|
||||
let _embedThrew = false;
|
||||
try {
|
||||
@@ -1596,6 +1605,23 @@ export function splitByTokenBudget(
|
||||
return batches;
|
||||
}
|
||||
|
||||
/**
|
||||
* Split a batch into sub-batches of at most `maxItems` inputs. Enforces a
|
||||
* hard COUNT cap that the token-budget split can't (many tiny inputs fit
|
||||
* under any token budget). Used for endpoints like llama.cpp's llama-server
|
||||
* that reject requests exceeding their launch batch size.
|
||||
*
|
||||
* @internal exported for tests; not part of the public gateway API.
|
||||
*/
|
||||
export function capBatchItems(texts: string[], maxItems: number): string[][] {
|
||||
if (maxItems <= 0 || texts.length <= maxItems) return [texts];
|
||||
const batches: string[][] = [];
|
||||
for (let i = 0; i < texts.length; i += maxItems) {
|
||||
batches.push(texts.slice(i, i + maxItems));
|
||||
}
|
||||
return batches;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if the error looks like a provider batch-token-limit error.
|
||||
*
|
||||
|
||||
@@ -35,9 +35,12 @@ export const llamaServer: Recipe = {
|
||||
trust_custom_dims: true, // #2271: user knows the launched model's native dim
|
||||
cost_per_1m_tokens_usd: 0,
|
||||
price_last_verified: '2026-05-10',
|
||||
// llama-server's batch capacity is set by `--ctx-size` at launch
|
||||
// time; no static cap to declare. v0.32 (#779).
|
||||
no_batch_cap: true,
|
||||
// llama-server enforces a hard request-COUNT cap equal to its launch
|
||||
// batch size (`--batch-size`, default 32): it rejects requests with
|
||||
// more inputs with `batch size N > maximum allowed batch size 32`.
|
||||
// The token-budget split can't bound item count, so cap it here. A
|
||||
// server launched with a larger `-b` can raise this. v0.32 (#779).
|
||||
max_batch_items: 32,
|
||||
},
|
||||
},
|
||||
/**
|
||||
|
||||
@@ -23,6 +23,14 @@ import type { Recipe } from '../types.ts';
|
||||
* envelope, not every individual model's capability. When in doubt about a
|
||||
* specific model, check https://openrouter.ai/models.
|
||||
*
|
||||
* Reranker: `/api/v1/rerank` proxies cross-encoder rerankers (Cohere v3.5/4-fast/4-pro
|
||||
* and NVIDIA Nemotron VL). Wire shape matches `gateway.rerank()`:
|
||||
* `{ query, documents, model }` → `{ results: [{ index, relevance_score }] }`.
|
||||
* Unlike embedding/chat, the reranker path strictly enforces the `models`
|
||||
* allowlist (no openai-compat bypass) — adding new rerank models requires a
|
||||
* recipe edit. Cohere bills per-search; the `cost_per_1m_tokens_usd` value
|
||||
* is a pseudo-rate for the budget tracker's `chars/4` heuristic.
|
||||
*
|
||||
* Attribution: OpenRouter recommends `HTTP-Referer` (required for app
|
||||
* attribution) + `X-OpenRouter-Title` (preferred; `X-Title` kept as
|
||||
* back-compat alias per OR docs). Defaults to `https://gbrain.ai` / `gbrain`;
|
||||
@@ -99,6 +107,30 @@ export const openrouter: Recipe = {
|
||||
// Let upstream errors surface per-model.
|
||||
price_last_verified: '2026-05-20',
|
||||
},
|
||||
reranker: {
|
||||
models: [
|
||||
'cohere/rerank-v3.5',
|
||||
'cohere/rerank-4-fast',
|
||||
'cohere/rerank-4-pro',
|
||||
'nvidia/llama-nemotron-rerank-vl-1b-v2:free',
|
||||
],
|
||||
default_model: 'cohere/rerank-v3.5',
|
||||
// Cohere bills per-search, not per-token. This is a pseudo-per-1M rate
|
||||
// for the budget tracker's heuristic (estimates tokens as chars/4).
|
||||
// At ~4K chars/search the tracker estimates ~$0.00025 — in the right
|
||||
// ballpark for the per-search bill. Patch budget-tracker.ts to honour a
|
||||
// `cost_per_search_usd` field for exact accounting.
|
||||
cost_per_1m_tokens_usd: 0.001,
|
||||
price_last_verified: '2026-06-13',
|
||||
// OpenRouter doesn't publish an explicit payload cap; 5MB matches
|
||||
// ZeroEntropy's upstream limit and the gateway's pre-flight ceiling.
|
||||
max_payload_bytes: 5_000_000,
|
||||
// OR serves /rerank under /api/v1. base_url_default already ends in /v1,
|
||||
// so gateway concatenates to …/api/v1/rerank.
|
||||
path: '/rerank',
|
||||
// OpenRouter rerank is fast (<200 ms p50); 5 s covers cold path safely.
|
||||
default_timeout_ms: 5_000,
|
||||
},
|
||||
},
|
||||
setup_hint:
|
||||
'Get an API key at https://openrouter.ai/settings/keys, then `export OPENROUTER_API_KEY=...` and use `openrouter:<provider>/<model>`. Optional overrides: OPENROUTER_BASE_URL (proxy), OPENROUTER_REFERER (attribution URL), OPENROUTER_TITLE (attribution name).',
|
||||
|
||||
@@ -54,6 +54,16 @@ export interface EmbeddingTouchpoint {
|
||||
* `max_batch_tokens` is also set.
|
||||
*/
|
||||
safety_factor?: number;
|
||||
/**
|
||||
* Maximum number of inputs per embedding request. Some endpoints enforce a
|
||||
* hard COUNT cap independent of token budget — notably llama.cpp's
|
||||
* `llama-server`, which rejects requests with more inputs than its launch
|
||||
* batch size (e.g. `batch size 100 > maximum allowed batch size 32`). The
|
||||
* token-budget pre-split cannot bound item count (many tiny chunks fit under
|
||||
* any token budget), so this is enforced as a separate hard re-split after
|
||||
* the token split. When unset, no count cap is applied.
|
||||
*/
|
||||
max_batch_items?: number;
|
||||
/**
|
||||
* v0.27.1: when true, at least one model in this recipe accepts image
|
||||
* inputs via a multimodal embedding endpoint (e.g. Voyage's
|
||||
|
||||
@@ -217,9 +217,19 @@ export function parseResolverEntries(resolverContent: string): ResolverEntry[] {
|
||||
// `skillsDir/*/SKILL.md` when manifest.json is missing — the scenario
|
||||
// needed for AGENTS.md-only OpenClaw deployments. See D-CX-12 / F-ENG-1.
|
||||
|
||||
/** Simple YAML frontmatter parser — extracts triggers array if present. */
|
||||
function extractTriggers(skillContent: string): string[] {
|
||||
const fmMatch = skillContent.match(/^---\n([\s\S]*?)\n---/);
|
||||
/**
|
||||
* Simple YAML frontmatter parser — extracts triggers array if present.
|
||||
*
|
||||
* Normalizes CRLF → LF before parsing so Windows checkouts (where
|
||||
* `core.autocrlf=true` is the default) parse correctly. Without this,
|
||||
* the `^---\n` and `^triggers:\s*\n` regexes never match because the
|
||||
* file content is `---\r\n` / `triggers:\r\n`, and every skill on
|
||||
* Windows is reported as `mece_gap` regardless of its actual content.
|
||||
* CI runs on Ubuntu-only so the bug only surfaces in user environments.
|
||||
*/
|
||||
export function extractTriggers(skillContent: string): string[] {
|
||||
const content = skillContent.replace(/\r\n/g, '\n');
|
||||
const fmMatch = content.match(/^---\n([\s\S]*?)\n---/);
|
||||
if (!fmMatch) return [];
|
||||
const fm = fmMatch[1];
|
||||
const triggersMatch = fm.match(/^triggers:\s*\n((?:\s+-\s+.+\n?)*)/m);
|
||||
|
||||
@@ -168,6 +168,15 @@ export interface CodeChunkOptions {
|
||||
largeChunkThresholdTokens?: number;
|
||||
fallbackChunkSizeWords?: number;
|
||||
fallbackOverlapWords?: number;
|
||||
/**
|
||||
* Hard upper bound (estimated tokens) on any single emitted chunk. A node
|
||||
* the AST splitter can't break up (a giant object/array literal, a single
|
||||
* huge assignment, a massive template literal) would otherwise be emitted
|
||||
* whole and rejected by the embedder ("input exceeds context length").
|
||||
* Chunks over this budget are recursively re-split. Default 2000 fits the
|
||||
* smallest common embedder context (e.g. nomic-embed-text, 2048).
|
||||
*/
|
||||
maxChunkTokens?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -549,6 +558,7 @@ export function parseWithTimeout(
|
||||
}
|
||||
|
||||
const DEFAULT_CHUNKER_TIMEOUT_MS = 30_000;
|
||||
const DEFAULT_MAX_CHUNK_TOKENS = 2000;
|
||||
|
||||
function resolveChunkerTimeoutMs(): number {
|
||||
const raw = process.env.GBRAIN_CHUNKER_TIMEOUT_MS;
|
||||
@@ -706,9 +716,9 @@ export async function chunkCodeTextFull(
|
||||
}
|
||||
|
||||
if (chunks.length === 0) {
|
||||
return { chunks: fallbackChunks(source, filePath, language, opts), edges: rawEdges };
|
||||
return { chunks: capOversizedChunks(fallbackChunks(source, filePath, language, opts), filePath, language, opts), edges: rawEdges };
|
||||
}
|
||||
return { chunks: mergeSmallSiblings(chunks, chunkTarget), edges: rawEdges };
|
||||
return { chunks: capOversizedChunks(mergeSmallSiblings(chunks, chunkTarget), filePath, language, opts), edges: rawEdges };
|
||||
} catch {
|
||||
return { chunks: fallbackChunks(source, filePath, language, opts), edges: [] };
|
||||
} finally {
|
||||
@@ -814,6 +824,73 @@ function buildMergedChunk(group: CodeChunk[], index: number): CodeChunk {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Final safety net: guarantee no emitted chunk exceeds the embedder's context
|
||||
* budget. tree-sitter splitting (splitLargeNode) can only break up a node that
|
||||
* exposes a `body` with >= 2 named children. A node without one — a giant
|
||||
* object/array literal, a single huge assignment, a massive template literal —
|
||||
* is emitted whole, producing a chunk far larger than the embedder accepts.
|
||||
* The embedder then rejects it ("input exceeds context length") and the chunk
|
||||
* is never embedded. Recursively re-split any over-budget chunk; fall back to a
|
||||
* hard character split for pathological no-whitespace content (e.g. a minified
|
||||
* one-liner) where word/line splitting can't get under budget.
|
||||
*/
|
||||
function capOversizedChunks(
|
||||
chunks: CodeChunk[],
|
||||
filePath: string,
|
||||
language: SupportedCodeLanguage,
|
||||
opts: CodeChunkOptions,
|
||||
): CodeChunk[] {
|
||||
const cap = opts.maxChunkTokens ?? DEFAULT_MAX_CHUNK_TOKENS;
|
||||
if (!chunks.some((c) => estimateTokens(c.text) > cap)) return chunks;
|
||||
const out: CodeChunk[] = [];
|
||||
for (const c of chunks) {
|
||||
if (estimateTokens(c.text) <= cap) {
|
||||
out.push({ ...c, index: out.length });
|
||||
continue;
|
||||
}
|
||||
// Strip the structured header ("[Lang] path:N-M symbol\n\n") so the splitter
|
||||
// works on the raw body; buildChunk re-adds a header to each piece.
|
||||
const body = c.text.replace(/^\[[^\]]+\] [^\n]+\n\n/, '');
|
||||
for (const piece of splitToTokenBudget(body, cap, opts)) {
|
||||
if (!piece.trim()) continue;
|
||||
out.push(buildChunk({
|
||||
body: piece,
|
||||
filePath,
|
||||
language,
|
||||
symbolName: c.metadata.symbolName,
|
||||
symbolType: c.metadata.symbolType,
|
||||
startLine: c.metadata.startLine,
|
||||
endLine: c.metadata.endLine,
|
||||
index: out.length,
|
||||
parentSymbolPath: c.metadata.parentSymbolPath,
|
||||
}));
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Split `text` into pieces each estimated <= cap tokens. Word/line-aware
|
||||
* (recursiveChunk) first; a hard character split is the last resort for
|
||||
* content with no whitespace to break on. */
|
||||
function splitToTokenBudget(text: string, cap: number, opts: CodeChunkOptions): string[] {
|
||||
const out: string[] = [];
|
||||
const pieces = recursiveChunk(text, {
|
||||
chunkSize: opts.fallbackChunkSizeWords ?? 300,
|
||||
chunkOverlap: opts.fallbackOverlapWords ?? 50,
|
||||
}).map((p) => p.text);
|
||||
for (const piece of pieces) {
|
||||
if (estimateTokens(piece) <= cap) {
|
||||
out.push(piece);
|
||||
continue;
|
||||
}
|
||||
// ~3.5 chars/token is a conservative cl100k estimate for source text.
|
||||
const charBudget = Math.max(1, Math.floor(cap * 3.5));
|
||||
for (let i = 0; i < piece.length; i += charBudget) out.push(piece.slice(i, i + charBudget));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------- Internals ----------
|
||||
|
||||
function fallbackChunks(
|
||||
|
||||
+33
-1
@@ -620,7 +620,10 @@ export function loadConfig(): GBrainConfig | null {
|
||||
* size the schema and must be stable across engine connect.
|
||||
*/
|
||||
export async function loadConfigWithEngine(
|
||||
engine: { getConfig(key: string): Promise<string | null | undefined> },
|
||||
engine: {
|
||||
getConfig(key: string): Promise<string | null | undefined>;
|
||||
listConfigKeys?(prefix: string): Promise<string[]>;
|
||||
},
|
||||
base?: GBrainConfig | null,
|
||||
): Promise<GBrainConfig | null> {
|
||||
// Codex /ship finding #3: when there's no file config AND no env DB URL,
|
||||
@@ -657,11 +660,31 @@ export async function loadConfigWithEngine(
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
async function dbPrefixMap(prefix: string): Promise<Record<string, string> | undefined> {
|
||||
if (typeof engine.listConfigKeys !== 'function') return undefined;
|
||||
let keys: string[];
|
||||
try {
|
||||
keys = await engine.listConfigKeys(prefix);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const out: Record<string, string> = {};
|
||||
for (const key of keys.sort()) {
|
||||
if (!key.startsWith(prefix)) continue;
|
||||
const leaf = key.slice(prefix.length);
|
||||
if (!leaf) continue;
|
||||
const value = await dbStr(key);
|
||||
if (value !== undefined) out[leaf] = value;
|
||||
}
|
||||
return Object.keys(out).length > 0 ? out : undefined;
|
||||
}
|
||||
|
||||
const dbMultimodal = await dbBool('embedding_multimodal');
|
||||
const dbMultimodalModel = await dbStr('embedding_multimodal_model');
|
||||
const dbOcr = await dbBool('embedding_image_ocr');
|
||||
const dbOcrModel = await dbStr('embedding_image_ocr_model');
|
||||
const dbProviderBaseUrls = await dbPrefixMap('provider_base_urls.');
|
||||
// v0.36 (D7) — embedding-column registry merge. Stored as JSON string in
|
||||
// the config table. Parse + shape-check here; full registry validation
|
||||
// (regex on keys, type/dim/provider field shapes) runs in the resolver at
|
||||
@@ -685,6 +708,15 @@ export async function loadConfigWithEngine(
|
||||
if (merged.embedding_image_ocr_model === undefined && dbOcrModel !== undefined) {
|
||||
merged.embedding_image_ocr_model = dbOcrModel;
|
||||
}
|
||||
if (dbProviderBaseUrls !== undefined) {
|
||||
const next = { ...(merged.provider_base_urls ?? {}) };
|
||||
for (const [providerId, baseUrl] of Object.entries(dbProviderBaseUrls)) {
|
||||
if (next[providerId] === undefined) next[providerId] = baseUrl;
|
||||
}
|
||||
if (Object.keys(next).length > 0) {
|
||||
merged.provider_base_urls = next;
|
||||
}
|
||||
}
|
||||
if (merged.embedding_columns === undefined && dbEmbeddingColumns !== undefined) {
|
||||
try {
|
||||
const parsed = JSON.parse(dbEmbeddingColumns);
|
||||
|
||||
@@ -90,7 +90,14 @@ export async function runExtractAtomsDrain(
|
||||
// Stop if a batch made zero forward progress — extraction is failing or
|
||||
// everything left is ineligible (e.g. all skipped). Prevents a hot loop
|
||||
// that spends budget without draining.
|
||||
if (r.extracted === 0 && r.skipped === 0) { stopped = 'no_progress'; break; }
|
||||
//
|
||||
// #2144: a zero-ATOM batch can still be progress — tombstoned
|
||||
// zero-yield pages shrink the backlog without producing atoms. Only
|
||||
// stop when the backlog count genuinely didn't move.
|
||||
if (r.extracted === 0 && r.skipped === 0) {
|
||||
const after = await deps.countRemaining();
|
||||
if (after === null || before === null || after >= before) { stopped = 'no_progress'; break; }
|
||||
}
|
||||
}
|
||||
|
||||
const remaining = await deps.countRemaining();
|
||||
|
||||
@@ -163,10 +163,20 @@ interface ExtractedAtom {
|
||||
body: string;
|
||||
source_quote?: string;
|
||||
lesson?: string;
|
||||
/**
|
||||
* 1-3 kebab-case topic labels for concept clustering. Consumed by
|
||||
* synthesize_concepts (groups atoms by `frontmatter.concepts`; only
|
||||
* labels shared by >=2 atoms materialize a concept page, so the prompt
|
||||
* biases reuse-over-coinage). #2123.
|
||||
*/
|
||||
concepts?: string[];
|
||||
virality_score?: number;
|
||||
emotional_register?: string;
|
||||
}
|
||||
|
||||
/** kebab-case validator for concept labels ("captive-portal", "channel-pricing"). */
|
||||
const CONCEPT_LABEL_RE = /^[a-z0-9]+(-[a-z0-9]+)*$/;
|
||||
|
||||
const EXTRACT_PROMPT = `You extract atomic content nuggets from a transcript.
|
||||
|
||||
An atom is a single-source, self-contained idea that could become a tweet,
|
||||
@@ -177,12 +187,17 @@ quote, or short essay angle. Each atom must:
|
||||
|
||||
Output a JSON array of atoms (1-3 per transcript, never more than 3).
|
||||
Each atom: {title (≤80 chars), atom_type, body (2-4 sentences),
|
||||
source_quote (verbatim ≤200 chars), lesson (one sentence), virality_score
|
||||
(0-100), emotional_register (one of: shocking, inspiring, funny, sobering,
|
||||
practical, controversial)}.
|
||||
source_quote (verbatim ≤200 chars), lesson (one sentence), concepts
|
||||
(1-3 topic labels), virality_score (0-100), emotional_register (one of:
|
||||
shocking, inspiring, funny, sobering, practical, controversial)}.
|
||||
|
||||
atom_type MUST be one of: ${ATOM_TYPES.join(', ')}.
|
||||
|
||||
concepts are kebab-case English TOPIC labels used to cluster atoms into
|
||||
concept pages (e.g. "captive-portal", "channel-pricing-strategy") — never
|
||||
entity or brand names. Use the same label for the same topic across atoms;
|
||||
prefer a label you already used over coining a near-synonym.
|
||||
|
||||
Output ONLY the JSON array, no prose.`;
|
||||
|
||||
interface DiscoveredPage {
|
||||
@@ -226,6 +241,7 @@ export async function discoverExtractablePages(
|
||||
AND COALESCE(p.frontmatter->>'imported_from', '') <> 'markdown-greenfield'
|
||||
AND COALESCE(p.frontmatter->>'dream_generated', '') <> 'true'
|
||||
AND length(COALESCE(p.compiled_truth, '')) >= $3
|
||||
AND COALESCE(p.frontmatter->>'atoms_scan_hash', '') <> substring(p.content_hash from 1 for 16)
|
||||
${hasFilter ? "AND p.slug = ANY($5::text[])" : ''}
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
@@ -298,6 +314,7 @@ export async function countExtractAtomsBacklog(
|
||||
AND COALESCE(p.frontmatter->>'imported_from', '') <> 'markdown-greenfield'
|
||||
AND COALESCE(p.frontmatter->>'dream_generated', '') <> 'true'
|
||||
AND length(COALESCE(p.compiled_truth, '')) >= $3
|
||||
AND COALESCE(p.frontmatter->>'atoms_scan_hash', '') <> substring(p.content_hash from 1 for 16)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM pages atom
|
||||
WHERE atom.type = 'atom' AND atom.source_id = $1
|
||||
@@ -311,6 +328,7 @@ export async function countExtractAtomsBacklog(
|
||||
AND COALESCE(p.frontmatter->>'imported_from', '') <> 'markdown-greenfield'
|
||||
AND COALESCE(p.frontmatter->>'dream_generated', '') <> 'true'
|
||||
AND length(COALESCE(p.compiled_truth, '')) >= $2
|
||||
AND COALESCE(p.frontmatter->>'atoms_scan_hash', '') <> substring(p.content_hash from 1 for 16)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM pages atom
|
||||
WHERE atom.type = 'atom' AND atom.source_id = p.source_id
|
||||
@@ -556,6 +574,25 @@ export async function runPhaseExtractAtoms(
|
||||
|
||||
const atoms = parseAtomsResponse(result.text);
|
||||
if (atoms.length === 0) {
|
||||
// #2144: tombstone zero-yield pages so they stop being rediscovered.
|
||||
// Idempotency is keyed on atom rows — a page that yields no atoms
|
||||
// leaves no row, so pre-fix it re-entered the discovery window every
|
||||
// run (wedging --drain with a false no_progress and re-spending
|
||||
// nightly budget on the same pages). Stamp the content hash we
|
||||
// scanned; discovery skips the page only while its content is
|
||||
// unchanged (edits re-eligibilize, mirroring atom-row staleness).
|
||||
// Only stamped after a SUCCESSFUL chat call — LLM failures take the
|
||||
// catch path below and stay retryable.
|
||||
if (!opts.dryRun && item.kind === 'page') {
|
||||
try {
|
||||
await engine.executeRaw(
|
||||
`UPDATE pages
|
||||
SET frontmatter = frontmatter || jsonb_build_object('atoms_scan_hash', $1::text)
|
||||
WHERE source_id = $2 AND slug = $3 AND deleted_at IS NULL`,
|
||||
[item.contentHash.slice(0, 16), sourceId, item.slug],
|
||||
);
|
||||
} catch { /* fail-soft: page stays rediscoverable */ }
|
||||
}
|
||||
if (item.kind === 'transcript') transcriptsProcessed++;
|
||||
else pagesProcessed++;
|
||||
continue;
|
||||
@@ -585,6 +622,7 @@ export async function runPhaseExtractAtoms(
|
||||
source_hash: item.contentHash.slice(0, 16),
|
||||
...(atom.source_quote && { source_quote: atom.source_quote }),
|
||||
...(atom.lesson && { lesson: atom.lesson }),
|
||||
...(atom.concepts && atom.concepts.length > 0 && { concepts: atom.concepts }),
|
||||
...(atom.virality_score !== undefined && { virality_score: atom.virality_score }),
|
||||
...(atom.emotional_register && { emotional_register: atom.emotional_register }),
|
||||
extracted_at: new Date().toISOString(),
|
||||
@@ -721,6 +759,13 @@ export function parseAtomsResponse(raw: string): ExtractedAtom[] {
|
||||
body,
|
||||
source_quote: typeof obj.source_quote === 'string' ? obj.source_quote.slice(0, 500) : undefined,
|
||||
lesson: typeof obj.lesson === 'string' ? obj.lesson : undefined,
|
||||
concepts: (() => {
|
||||
if (!Array.isArray(obj.concepts)) return undefined;
|
||||
const labels = obj.concepts
|
||||
.filter((c): c is string => typeof c === 'string' && CONCEPT_LABEL_RE.test(c))
|
||||
.slice(0, 3);
|
||||
return labels.length > 0 ? labels : undefined;
|
||||
})(),
|
||||
virality_score:
|
||||
typeof obj.virality_score === 'number' &&
|
||||
obj.virality_score >= 0 &&
|
||||
|
||||
@@ -1218,11 +1218,21 @@ export interface BrainEngine {
|
||||
*
|
||||
* Uses the `%` trigram operator (GIN-indexed) + the standard `similarity()`
|
||||
* function. Both engines support pg_trgm (PGLite 0.3+, Postgres always).
|
||||
*
|
||||
* `sourceId` constrains the search to a single source and filters out
|
||||
* soft-deleted pages. Mirrors the same filters `tryFuzzyMatch` in
|
||||
* `src/core/entities/resolve.ts` got via #1436 (v0.41.13.0). Omit for the
|
||||
* historical unscoped behavior — live-mode callers that already know
|
||||
* the source should pass it to avoid cross-source slug suggestions that
|
||||
* get silently dropped at the FK filter downstream. Batch-mode callers
|
||||
* (e.g. `gbrain extract`) intentionally omit it to build a cross-source
|
||||
* resolution map.
|
||||
*/
|
||||
findByTitleFuzzy(
|
||||
name: string,
|
||||
dirPrefix?: string,
|
||||
minSimilarity?: number,
|
||||
sourceId?: string,
|
||||
): Promise<{ slug: string; similarity: number } | null>;
|
||||
/**
|
||||
* v0.34.1 (#861 — P0 leak seal): `opts.sourceId` / `opts.sourceIds`
|
||||
|
||||
@@ -79,11 +79,11 @@ export type LinkResolutionType = 'qualified' | 'unqualified';
|
||||
/**
|
||||
* Directory prefix whitelist. These are the top-level slug dirs the extractor
|
||||
* recognizes as entity references. Upstream canonical + our extensions:
|
||||
* - Gbrain canonical: people, companies, meetings, concepts, deal, civic, project, source, media, yc, projects
|
||||
* - Gbrain canonical: people, companies, meetings, concepts, deal, civic, project, source, media, yc, projects, reference
|
||||
* - Our domain extensions: tech, finance, personal, openclaw (domain-organized wikis)
|
||||
* - Our entity prefix: entities (we kept some legacy entities/projects/ pages)
|
||||
*/
|
||||
const DIR_PATTERN = '(?:people|companies|meetings|concepts|deal|civic|project|projects|source|media|yc|tech|finance|personal|openclaw|entities)';
|
||||
const DIR_PATTERN = '(?:people|companies|meetings|concepts|deal|civic|project|projects|source|media|yc|tech|finance|personal|openclaw|entities|reference)';
|
||||
|
||||
/**
|
||||
* Match `[Name](path)` markdown links pointing to entity directories.
|
||||
@@ -980,10 +980,14 @@ export function makeResolver(
|
||||
|
||||
// Step 3: pg_trgm fuzzy title match — both modes. Tries each hint in
|
||||
// order; first hint with a ≥0.55 similarity match wins. If no hints,
|
||||
// try the whole pages table.
|
||||
// try the whole pages table. When opts.sourceId is set, the fuzzy
|
||||
// search is constrained to that source (and skips soft-deleted pages)
|
||||
// so cross-source slug suggestions don't get silently dropped at the
|
||||
// FK filter downstream. Mirrors the same scope fix `tryFuzzyMatch` got
|
||||
// via #1436.
|
||||
const searchHints = hints.length > 0 ? hints : [undefined];
|
||||
for (const hint of searchHints) {
|
||||
const match = await engine.findByTitleFuzzy(trimmed, hint, 0.55);
|
||||
const match = await engine.findByTitleFuzzy(trimmed, hint, 0.55, opts.sourceId);
|
||||
if (match) {
|
||||
cache.set(cacheKey, match.slug);
|
||||
return match.slug;
|
||||
|
||||
@@ -87,6 +87,11 @@ function walkMarkdownAndMdxFiles(
|
||||
for (const entry of entries) {
|
||||
if (truncated) return;
|
||||
if (entry.startsWith('.')) continue;
|
||||
// Skip heavy non-content dirs so the walk doesn't exhaust the time
|
||||
// budget on dependency/build trees (node_modules can be 50k+ files
|
||||
// with zero .md). These are never gbrain page sources.
|
||||
if (entry === 'node_modules' || entry === 'dist' || entry === 'build' ||
|
||||
entry === '.next' || entry === 'vendor' || entry === 'target') continue;
|
||||
const full = join(d, entry);
|
||||
let isDir = false;
|
||||
try {
|
||||
@@ -95,6 +100,9 @@ function walkMarkdownAndMdxFiles(
|
||||
continue;
|
||||
}
|
||||
if (isDir) {
|
||||
// Time check on directory descent too, so a deep dependency-free
|
||||
// tree still respects the deadline even before any .md is found.
|
||||
if (Date.now() >= deadlineMs) { truncated = true; return; }
|
||||
walk(full);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -1153,7 +1153,11 @@ async function runAutoLink(
|
||||
|
||||
// Live-mode resolver: per-put throwaway cache, pg_trgm + optional search.
|
||||
// Issue #972 (codex [P1]): pass sourceId so basename resolution stays
|
||||
// within this page's source — no cross-source basename edges.
|
||||
// within this page's source — no cross-source basename edges. Also scopes
|
||||
// the fuzzy fallback (findByTitleFuzzy) to the same source the put_page is
|
||||
// targeting — without it, cross-source slug suggestions get silently dropped
|
||||
// at the FK filter and the link looks like it failed to resolve. Twin of
|
||||
// #1436's `tryFuzzyMatch` fix.
|
||||
const resolver = makeResolver(engine, { mode: 'live', sourceId: opts?.sourceId });
|
||||
// Issue #972: opt-in bare-wikilink basename resolution. Off by default.
|
||||
const globalBasename = await isGlobalBasenameEnabled(engine);
|
||||
|
||||
@@ -1060,6 +1060,16 @@ export class PGLiteEngine implements BrainEngine {
|
||||
RETURNING id, source_id, slug, type, title, compiled_truth, timeline, frontmatter, content_hash, created_at, updated_at, effective_date, effective_date_source, import_filename, source_kind, source_uri, ingested_via, ingested_at`,
|
||||
[sourceId, slug, page.type, pageKind, page.title, page.compiled_truth, page.timeline || '', JSON.stringify(frontmatter), hash, effectiveDate, effectiveDateSource, importFilename, chunkerVersion, sourcePath, sourceKind, sourceUri, ingestedVia, ingestedAt]
|
||||
);
|
||||
// PGLite can return zero rows from INSERT ... ON CONFLICT DO UPDATE ...
|
||||
// RETURNING in no-op/trigger edge cases, which made rowToPage(undefined)
|
||||
// throw "undefined is not an object (evaluating 'row.deleted_at')" and
|
||||
// skip the file during sync. The row WAS written, so re-read instead of
|
||||
// crashing.
|
||||
if (rows.length === 0) {
|
||||
const reread = await this.getPage(slug, { sourceId });
|
||||
if (reread) return reread;
|
||||
throw new Error(`putPage: RETURNING produced no row for ${sourceId}/${slug}`);
|
||||
}
|
||||
return rowToPage(rows[0] as Record<string, unknown>);
|
||||
}
|
||||
|
||||
@@ -2912,22 +2922,41 @@ export class PGLiteEngine implements BrainEngine {
|
||||
name: string,
|
||||
dirPrefix?: string,
|
||||
minSimilarity: number = 0.55,
|
||||
sourceId?: string,
|
||||
): Promise<{ slug: string; similarity: number } | null> {
|
||||
// Inline threshold comparison instead of `SET LOCAL pg_trgm.similarity_threshold`.
|
||||
// The GUC only scopes to the current transaction and pglite auto-commits each
|
||||
// .query() call, so the SET LOCAL would be a no-op. Using similarity() >= $N
|
||||
// directly gives predictable behavior. Tie-breaker: sort by slug so re-runs
|
||||
// pick the same winner.
|
||||
//
|
||||
// `sourceId` + `deleted_at IS NULL` mirror the filters `tryFuzzyMatch` in
|
||||
// `src/core/entities/resolve.ts` got via #1436 (v0.41.13.0). Without them,
|
||||
// fuzzy resolution could suggest cross-source slugs that the caller then
|
||||
// silently drops at the FK filter — making it look like the match failed
|
||||
// when in fact it picked the wrong page.
|
||||
const prefixPattern = dirPrefix ? `${dirPrefix}/%` : '%';
|
||||
const { rows } = await this.db.query(
|
||||
`SELECT slug, similarity(title, $1) AS sim
|
||||
FROM pages
|
||||
WHERE similarity(title, $1) >= $3
|
||||
AND slug LIKE $2
|
||||
ORDER BY sim DESC, slug ASC
|
||||
LIMIT 1`,
|
||||
[name, prefixPattern, minSimilarity]
|
||||
);
|
||||
const { rows } = sourceId
|
||||
? await this.db.query(
|
||||
`SELECT slug, similarity(title, $1) AS sim
|
||||
FROM pages
|
||||
WHERE similarity(title, $1) >= $3
|
||||
AND slug LIKE $2
|
||||
AND source_id = $4
|
||||
AND deleted_at IS NULL
|
||||
ORDER BY sim DESC, slug ASC
|
||||
LIMIT 1`,
|
||||
[name, prefixPattern, minSimilarity, sourceId]
|
||||
)
|
||||
: await this.db.query(
|
||||
`SELECT slug, similarity(title, $1) AS sim
|
||||
FROM pages
|
||||
WHERE similarity(title, $1) >= $3
|
||||
AND slug LIKE $2
|
||||
ORDER BY sim DESC, slug ASC
|
||||
LIMIT 1`,
|
||||
[name, prefixPattern, minSimilarity]
|
||||
);
|
||||
if (rows.length === 0) return null;
|
||||
const row = rows[0] as { slug: string; sim: number };
|
||||
return { slug: row.slug, similarity: row.sim };
|
||||
@@ -5837,6 +5866,11 @@ export class PGLiteEngine implements BrainEngine {
|
||||
params.push(escaped);
|
||||
prefixCondition = `AND p.slug LIKE $${params.length} ESCAPE '\\'`;
|
||||
}
|
||||
// TIM-37: exclude briefing pages from their own Brain Pulse. See the
|
||||
// matching block in postgres-engine.ts getRecentSalience() for context.
|
||||
const excludeBriefings = !(slugPrefix && slugPrefix.startsWith('briefings'))
|
||||
? `AND p.slug NOT LIKE 'briefings/%'`
|
||||
: '';
|
||||
params.push(limit);
|
||||
const limitParam = `$${params.length}`;
|
||||
|
||||
@@ -5872,6 +5906,7 @@ export class PGLiteEngine implements BrainEngine {
|
||||
LEFT JOIN takes t ON t.page_id = p.id AND t.active = TRUE
|
||||
WHERE GREATEST(p.updated_at, COALESCE(p.salience_touched_at, p.updated_at)) >= $1::timestamptz
|
||||
${prefixCondition}
|
||||
${excludeBriefings}
|
||||
GROUP BY p.id
|
||||
ORDER BY score DESC
|
||||
LIMIT ${limitParam}`,
|
||||
|
||||
@@ -3082,6 +3082,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
name: string,
|
||||
dirPrefix?: string,
|
||||
minSimilarity: number = 0.55,
|
||||
sourceId?: string,
|
||||
): Promise<{ slug: string; similarity: number } | null> {
|
||||
const sql = this.sql;
|
||||
// Use the `similarity()` function directly with an explicit threshold
|
||||
@@ -3094,15 +3095,33 @@ export class PostgresEngine implements BrainEngine {
|
||||
// Tie-breaker: sort by slug after similarity so re-runs return the
|
||||
// same winner when multiple pages score equally (prevents churn
|
||||
// in put_page auto-link reconciliation).
|
||||
//
|
||||
// `sourceId` + `deleted_at IS NULL` mirror the filters `tryFuzzyMatch`
|
||||
// in `src/core/entities/resolve.ts` got via #1436 (v0.41.13.0). Without
|
||||
// them, fuzzy resolution could suggest cross-source slugs that the
|
||||
// caller then silently drops at the FK filter in
|
||||
// `operations.ts:reconcileLinks` (the `allSlugs` filter) — making it
|
||||
// look like the match failed when in fact it picked the wrong page.
|
||||
const prefixPattern = dirPrefix ? `${dirPrefix}/%` : '%';
|
||||
const rows = await sql`
|
||||
SELECT slug, similarity(title, ${name}) AS sim
|
||||
FROM pages
|
||||
WHERE similarity(title, ${name}) >= ${minSimilarity}
|
||||
AND slug LIKE ${prefixPattern}
|
||||
ORDER BY sim DESC, slug ASC
|
||||
LIMIT 1
|
||||
`;
|
||||
const rows = sourceId
|
||||
? await sql`
|
||||
SELECT slug, similarity(title, ${name}) AS sim
|
||||
FROM pages
|
||||
WHERE similarity(title, ${name}) >= ${minSimilarity}
|
||||
AND slug LIKE ${prefixPattern}
|
||||
AND source_id = ${sourceId}
|
||||
AND deleted_at IS NULL
|
||||
ORDER BY sim DESC, slug ASC
|
||||
LIMIT 1
|
||||
`
|
||||
: await sql`
|
||||
SELECT slug, similarity(title, ${name}) AS sim
|
||||
FROM pages
|
||||
WHERE similarity(title, ${name}) >= ${minSimilarity}
|
||||
AND slug LIKE ${prefixPattern}
|
||||
ORDER BY sim DESC, slug ASC
|
||||
LIMIT 1
|
||||
`;
|
||||
if (rows.length === 0) return null;
|
||||
const row = rows[0] as { slug: string; sim: number };
|
||||
return { slug: row.slug, similarity: row.sim };
|
||||
@@ -6153,6 +6172,13 @@ export class PostgresEngine implements BrainEngine {
|
||||
const prefixCondition = slugPrefix
|
||||
? sql`AND p.slug LIKE ${slugPrefix.replace(/[\\%_]/g, (c) => '\\' + c) + '%'} ESCAPE '\\'`
|
||||
: sql``;
|
||||
// TIM-37: exclude briefing pages from their own Brain Pulse. The cron
|
||||
// briefing writes to 90_Briefings/, gets re-ingested, and would otherwise
|
||||
// top tomorrow's salience as pure self-reference. Suppress unless the
|
||||
// caller explicitly asked for the briefings/ prefix.
|
||||
const excludeBriefings = !(slugPrefix && slugPrefix.startsWith('briefings'))
|
||||
? sql`AND p.slug NOT LIKE 'briefings/%'`
|
||||
: sql``;
|
||||
// v0.29.1: third score term via buildRecencyComponentSql. Default
|
||||
// 'flat' = v0.29.0 behavior (1 / (1 + days_old)). 'on' opts into the
|
||||
// per-prefix decay map (concepts/ evergreen, daily/ aggressive, etc.).
|
||||
@@ -6186,6 +6212,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
LEFT JOIN takes t ON t.page_id = p.id AND t.active = TRUE
|
||||
WHERE GREATEST(p.updated_at, COALESCE(p.salience_touched_at, p.updated_at)) >= ${boundaryIso}::timestamptz
|
||||
${prefixCondition}
|
||||
${excludeBriefings}
|
||||
GROUP BY p.id
|
||||
ORDER BY score DESC
|
||||
LIMIT ${limit}
|
||||
|
||||
+35
-1
@@ -556,6 +556,40 @@ export async function runThink(
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip a "## Gaps" section from an answer body.
|
||||
*
|
||||
* `think` returns gaps in the structured `gaps` array, which the CLI and the
|
||||
* persisted synthesis page render exactly once. The system prompt also used to
|
||||
* ask for a "Gaps" section inside the answer prose, so a model that still emits
|
||||
* one would make the output show "## Gaps" twice — once from the prose, once
|
||||
* from the structured array. This removes the prose section so the structured
|
||||
* array stays the single source of truth.
|
||||
*
|
||||
* Matches a heading line `## Gaps` (level 2-6, case-insensitive) and removes it
|
||||
* through the next heading of the same-or-higher level, or end of string.
|
||||
* Returns the input unchanged when there is no such section.
|
||||
*/
|
||||
export function stripGapsSection(answer: string): string {
|
||||
if (!answer) return answer;
|
||||
const lines = answer.split('\n');
|
||||
let start = -1;
|
||||
let level = 0;
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const m = /^(#{2,6})\s+gaps\s*$/i.exec(lines[i]);
|
||||
if (m) { start = i; level = m[1].length; break; }
|
||||
}
|
||||
if (start === -1) return answer;
|
||||
let end = lines.length;
|
||||
for (let i = start + 1; i < lines.length; i++) {
|
||||
const h = /^(#{1,6})\s+\S/.exec(lines[i]);
|
||||
if (h && h[1].length <= level) { end = i; break; }
|
||||
}
|
||||
const kept = [...lines.slice(0, start), ...lines.slice(end)].join('\n');
|
||||
// Drop trailing blank lines left by removing a trailing section.
|
||||
return kept.replace(/\s+$/, '');
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist a synthesis page + its evidence. Returns the saved slug.
|
||||
* Synthesis pages are written under `synthesis/<slugified-question>-<date>.md`.
|
||||
@@ -585,7 +619,7 @@ export async function persistSynthesis(
|
||||
const body = [
|
||||
`# ${result.question}`,
|
||||
'',
|
||||
result.answer,
|
||||
stripGapsSection(result.answer),
|
||||
'',
|
||||
result.gaps.length > 0 ? '## Gaps\n\n' + result.gaps.map(g => `- ${g}`).join('\n') : '',
|
||||
].filter(Boolean).join('\n');
|
||||
|
||||
@@ -52,19 +52,19 @@ Hard rules:
|
||||
rather than asserting it as established. Confidence is part of the data.
|
||||
- If two takes contradict (different holders, opposite claims), surface BOTH in a "Conflicts"
|
||||
section. Never silently pick one.
|
||||
- If you cannot answer because the brain doesn't contain the relevant data, say so in the
|
||||
"Gaps" section. List the specific missing pieces. Do not make up answers.
|
||||
- If the brain doesn't contain data needed to answer, do NOT make it up. Record each
|
||||
missing piece in the structured "gaps" array (below), not as a section in the answer prose.
|
||||
- Never instruct the user (no "you should" / "I recommend X"). The brain reports; the user decides.
|
||||
- Output MUST be valid JSON matching the schema below. No prose outside JSON.
|
||||
|
||||
Output schema:
|
||||
{
|
||||
"answer": "<markdown body. Inline citations like [slug#row] or [slug]. Sections: Answer, Conflicts (optional), Gaps>",
|
||||
"answer": "<markdown body. Inline citations like [slug#row] or [slug]. Sections: Answer, Conflicts (optional). Do NOT add a Gaps section here — gaps belong in the gaps array.>",
|
||||
"citations": [
|
||||
{"page_slug": "people/alice-example", "row_num": 3, "citation_index": 1},
|
||||
{"page_slug": "companies/acme-example", "row_num": null, "citation_index": 2}
|
||||
],
|
||||
"gaps": ["specific missing data point 1", "specific missing data point 2"]
|
||||
"gaps": ["a specific, self-contained missing-or-stale data point, citing the [slug] where relevant", "another specific gap"]
|
||||
}
|
||||
|
||||
The "row_num" field is required for take citations and MUST be null for page-only citations.`;
|
||||
@@ -83,7 +83,7 @@ export function buildThinkSystemPrompt(opts: ThinkSystemPromptOpts = {}): string
|
||||
lines.push(`\nThis is a temporal question. Order key claims chronologically when it helps the reader.`);
|
||||
}
|
||||
if (opts.willSave) {
|
||||
lines.push(`\nThis synthesis will be persisted as a brain page. Aim for completeness — cover Answer, Conflicts, and Gaps thoroughly.`);
|
||||
lines.push(`\nThis synthesis will be persisted as a brain page. Aim for completeness — cover the Answer and any Conflicts thoroughly, and list every missing piece in the structured "gaps" array.`);
|
||||
}
|
||||
if (opts.withCalibration) {
|
||||
lines.push(
|
||||
@@ -92,7 +92,7 @@ export function buildThinkSystemPrompt(opts: ThinkSystemPromptOpts = {}): string
|
||||
lines.push(`- Name both the user's PRIOR (default reasoning) AND the COUNTER-PRIOR from their hedged-domain self.`);
|
||||
lines.push(`- Reference active bias tags by name when relevant ("this fits the over-confident-geography pattern").`);
|
||||
lines.push(`- Do NOT silently substitute the debiased answer. ALWAYS surface both priors transparently.`);
|
||||
lines.push(`- Track-record sentences belong in a "Calibration" section in the answer body, between Conflicts and Gaps.`);
|
||||
lines.push(`- Track-record sentences belong in a "Calibration" section in the answer body, after the Conflicts section (if present).`);
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
@@ -34,6 +34,7 @@ import {
|
||||
resetGateway,
|
||||
embed,
|
||||
splitByTokenBudget,
|
||||
capBatchItems,
|
||||
isTokenLimitError,
|
||||
__setEmbedTransportForTests,
|
||||
__getShrinkStateForTests,
|
||||
@@ -151,6 +152,41 @@ describe('splitByTokenBudget (pure helper)', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('capBatchItems (hard COUNT cap helper)', () => {
|
||||
test('batch at or under the cap is returned as a single batch (no copy of contents)', () => {
|
||||
const texts = ['a', 'b', 'c'];
|
||||
expect(capBatchItems(texts, 3)).toEqual([texts]);
|
||||
expect(capBatchItems(texts, 10)).toEqual([texts]);
|
||||
});
|
||||
|
||||
test('oversized batch splits into chunks of at most maxItems', () => {
|
||||
const texts = Array.from({ length: 100 }, (_, i) => `t${i}`);
|
||||
const result = capBatchItems(texts, 32);
|
||||
expect(result.map(b => b.length)).toEqual([32, 32, 32, 4]);
|
||||
expect(result.every(b => b.length <= 32)).toBe(true);
|
||||
});
|
||||
|
||||
test('exact multiple splits evenly with no trailing empty batch', () => {
|
||||
const texts = Array.from({ length: 64 }, (_, i) => `t${i}`);
|
||||
expect(capBatchItems(texts, 32).map(b => b.length)).toEqual([32, 32]);
|
||||
});
|
||||
|
||||
test('order is preserved across the split (concatenation round-trips)', () => {
|
||||
const texts = Array.from({ length: 70 }, (_, i) => `t${i}`);
|
||||
expect(capBatchItems(texts, 32).flat()).toEqual(texts);
|
||||
});
|
||||
|
||||
test('maxItems <= 0 is a no-op (single batch) — never produces empty/infinite batches', () => {
|
||||
const texts = ['a', 'b', 'c'];
|
||||
expect(capBatchItems(texts, 0)).toEqual([texts]);
|
||||
expect(capBatchItems(texts, -5)).toEqual([texts]);
|
||||
});
|
||||
|
||||
test('empty input returns a single empty batch', () => {
|
||||
expect(capBatchItems([], 32)).toEqual([[]]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('isTokenLimitError (pure helper)', () => {
|
||||
test('matches Voyage error format', () => {
|
||||
expect(isTokenLimitError(VOYAGE_TOKEN_LIMIT_ERROR)).toBe(true);
|
||||
|
||||
@@ -28,8 +28,8 @@ describe('v0.32 #779: no_batch_cap suppresses the missing-max_batch_tokens warni
|
||||
resetGateway();
|
||||
});
|
||||
|
||||
test('Ollama, LiteLLM, llama-server all declare no_batch_cap: true', () => {
|
||||
for (const id of ['ollama', 'litellm', 'llama-server']) {
|
||||
test('Ollama, LiteLLM declare no_batch_cap: true', () => {
|
||||
for (const id of ['ollama', 'litellm']) {
|
||||
const r = getRecipe(id);
|
||||
expect(r, `${id} not registered`).toBeDefined();
|
||||
expect(
|
||||
@@ -39,6 +39,16 @@ describe('v0.32 #779: no_batch_cap suppresses the missing-max_batch_tokens warni
|
||||
}
|
||||
});
|
||||
|
||||
test('llama-server declares a hard item-count cap (max_batch_items: 32)', () => {
|
||||
// llama.cpp enforces a request-COUNT cap equal to its launch --batch-size
|
||||
// (default 32); declaring max_batch_items both bounds batches AND suppresses
|
||||
// the missing-max_batch_tokens warning. Replaces the prior no_batch_cap flag.
|
||||
const r = getRecipe('llama-server');
|
||||
expect(r, 'llama-server not registered').toBeDefined();
|
||||
expect(r!.touchpoints.embedding?.max_batch_items).toBe(32);
|
||||
expect(r!.touchpoints.embedding?.no_batch_cap).toBeUndefined();
|
||||
});
|
||||
|
||||
test('configureGateway does NOT warn for ollama/litellm/llama-server', () => {
|
||||
warnSpy.mockClear();
|
||||
resetGateway();
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
/**
|
||||
* Ollama Matryoshka dims passthrough.
|
||||
*
|
||||
* Several embedding models served via Ollama (Qwen3-Embedding family) support
|
||||
* Matryoshka truncation through the `dimensions` field on /v1/embeddings.
|
||||
* Without this passthrough, gbrain ignores user-selected reduced dims and the
|
||||
* provider returns its native size, causing dim-mismatch failures against
|
||||
* brains configured for smaller widths.
|
||||
*/
|
||||
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { dimsProviderOptions } from '../../src/core/ai/dims.ts';
|
||||
|
||||
describe('dims: ollama Matryoshka models', () => {
|
||||
test('qwen3-embedding:4b threads dimensions=1536', () => {
|
||||
expect(dimsProviderOptions('openai-compatible', 'qwen3-embedding:4b', 1536))
|
||||
.toEqual({ openaiCompatible: { dimensions: 1536 } });
|
||||
});
|
||||
|
||||
test('qwen3-embedding:0.6b threads dimensions=512', () => {
|
||||
expect(dimsProviderOptions('openai-compatible', 'qwen3-embedding:0.6b', 512))
|
||||
.toEqual({ openaiCompatible: { dimensions: 512 } });
|
||||
});
|
||||
|
||||
test('qwen3-embedding:8b threads dimensions=2048', () => {
|
||||
expect(dimsProviderOptions('openai-compatible', 'qwen3-embedding:8b', 2048))
|
||||
.toEqual({ openaiCompatible: { dimensions: 2048 } });
|
||||
});
|
||||
|
||||
test('bare qwen3-embedding (no quant tag) also recognized', () => {
|
||||
expect(dimsProviderOptions('openai-compatible', 'qwen3-embedding', 1024))
|
||||
.toEqual({ openaiCompatible: { dimensions: 1024 } });
|
||||
});
|
||||
|
||||
test('unrelated openai-compat model returns undefined (regression guard)', () => {
|
||||
expect(dimsProviderOptions('openai-compatible', 'nomic-embed-text', 768))
|
||||
.toBeUndefined();
|
||||
expect(dimsProviderOptions('openai-compatible', 'mxbai-embed-large', 1024))
|
||||
.toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -99,3 +99,29 @@ describe('autopilot wrapper script — env source order (v0.36.1.x #966)', () =>
|
||||
expect(src).toMatch(/source\s+~\/\.zshrc/);
|
||||
});
|
||||
});
|
||||
|
||||
// v0.42.x: the wrapper must export PATH with ~/.bun/bin before exec'ing
|
||||
// gbrain. The exec'd gbrain has a `#!/usr/bin/env bun` shebang, and the
|
||||
// standard Debian ~/.bashrc ships a non-interactive guard
|
||||
// (`case $- in *i*) ;; *) return;; esac`) that exits early when cron/launchd/
|
||||
// systemd invokes bash non-interactively — so the PATH exports that
|
||||
// operators put in ~/.bashrc never reach this subprocess. Without the
|
||||
// explicit export the wrapper silently dies with `env: bun: No such file
|
||||
// or directory`, leaves a stale lockfile, and blocks every subsequent tick
|
||||
// for the 10-min stale-lock window. Regression: see Hermes `cron doctor`
|
||||
// reports — this caused a 1-week nightly-cycle outage on at least one
|
||||
// operator machine before being diagnosed.
|
||||
describe('autopilot wrapper script — bun PATH export (v0.42.x regression)', () => {
|
||||
test('wrapper exports ~/.bun/bin onto PATH before the exec', async () => {
|
||||
const { readFileSync } = await import('fs');
|
||||
const src = readFileSync('src/commands/autopilot.ts', 'utf8');
|
||||
// The export line must appear inside the writeWrapperScript heredoc.
|
||||
expect(src).toMatch(/export\s+PATH="\$HOME\/\.bun\/bin:\$PATH"/);
|
||||
// The export must precede the exec line, otherwise env never sees it.
|
||||
const exportIdx = src.search(/export\s+PATH="\$HOME\/\.bun\/bin/);
|
||||
const execIdx = src.search(/exec\s+'\${safeGbrainPath}'/);
|
||||
expect(exportIdx).toBeGreaterThan(0);
|
||||
expect(execIdx).toBeGreaterThan(0);
|
||||
expect(exportIdx).toBeLessThan(execIdx);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
/**
|
||||
* Structural regression for the backlinks Minion handler default.
|
||||
*
|
||||
* Backlinks jobs submitted with an EMPTY payload (the sync→embed→backlinks
|
||||
* chains enqueued after every ingestion) must run as 'check', never 'fix'.
|
||||
* The pre-fix handler inverted the default (`=== 'check' ? 'check' : 'fix'`),
|
||||
* so every routine post-ingestion job rewrote tracked brain pages with
|
||||
* generated "Referenced in" timeline bullets — contradicting the documented
|
||||
* intent in src/core/cycle.ts (runPhaseBacklinks): "Maintenance cycles must
|
||||
* not rewrite tracked brain pages with generated 'Referenced in' timeline
|
||||
* bullets."
|
||||
*
|
||||
* Source-grep is the right tool here (see fix-wave-structural.test.ts): the
|
||||
* handler dynamically imports runBacklinksCore and walks a real repo dir, so
|
||||
* a behavioral test would require heavy mocking that hides the regression
|
||||
* behind a test seam. The rule is "this specific default must stay 'check'".
|
||||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { readFileSync } from 'fs';
|
||||
|
||||
describe('backlinks Minion handler — empty payload defaults to check, not fix', () => {
|
||||
const src = readFileSync('src/commands/jobs.ts', 'utf8');
|
||||
|
||||
// Isolate the backlinks register block so assertions can't accidentally
|
||||
// match another handler's action parsing.
|
||||
const blockMatch = src.match(
|
||||
/worker\.register\('backlinks',[\s\S]*?runBacklinksCore\(\{[\s\S]*?\}\);/
|
||||
);
|
||||
|
||||
test('the backlinks handler block exists', () => {
|
||||
expect(blockMatch).not.toBeNull();
|
||||
});
|
||||
|
||||
test("default action is 'check' (explicit opt-in required for 'fix')", () => {
|
||||
const block = blockMatch![0];
|
||||
expect(block).toMatch(
|
||||
/job\.data\.action\s*===\s*'fix'\s*\?\s*'fix'\s*:\s*'check'/
|
||||
);
|
||||
});
|
||||
|
||||
test('the inverted (fix-by-default) shape stays absent', () => {
|
||||
const block = blockMatch![0];
|
||||
expect(block).not.toMatch(
|
||||
/job\.data\.action\s*===\s*'check'\s*\?\s*'check'\s*:\s*'fix'/
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
checkResolvable,
|
||||
parseResolverEntries,
|
||||
extractDelegationTargets,
|
||||
extractTriggers,
|
||||
} from "../src/core/check-resolvable.ts";
|
||||
|
||||
const SKILLS_DIR = join(import.meta.dir, "..", "skills");
|
||||
@@ -195,6 +196,39 @@ describe("parseResolverEntries", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("extractTriggers", () => {
|
||||
const LF_FRONTMATTER =
|
||||
"---\nname: query\ndescription: Test\ntriggers:\n - \"what do we know\"\n - \"tell me about\"\ntools:\n - search\n---\n\n# Body\n";
|
||||
|
||||
test("parses triggers from LF-terminated frontmatter", () => {
|
||||
const triggers = extractTriggers(LF_FRONTMATTER);
|
||||
expect(triggers).toEqual(["what do we know", "tell me about"]);
|
||||
});
|
||||
|
||||
test("parses triggers from CRLF-terminated frontmatter (Windows checkouts)", () => {
|
||||
// Regression: `core.autocrlf=true` is the Windows default. Without
|
||||
// CRLF→LF normalization, every Windows skill is reported as a false
|
||||
// mece_gap warning because the `^---\n` regex never matches `---\r\n`.
|
||||
const crlf = LF_FRONTMATTER.replace(/\n/g, "\r\n");
|
||||
const triggers = extractTriggers(crlf);
|
||||
expect(triggers).toEqual(["what do we know", "tell me about"]);
|
||||
});
|
||||
|
||||
test("returns [] when frontmatter is missing", () => {
|
||||
expect(extractTriggers("# Just a body, no frontmatter\n")).toEqual([]);
|
||||
});
|
||||
|
||||
test("returns [] when triggers field is absent from frontmatter", () => {
|
||||
const fm = "---\nname: query\ndescription: Test\ntools:\n - search\n---\n";
|
||||
expect(extractTriggers(fm)).toEqual([]);
|
||||
});
|
||||
|
||||
test("strips surrounding quotes from trigger values", () => {
|
||||
const fm = "---\nname: x\ntriggers:\n - \"double quoted\"\n - 'single quoted'\n - unquoted\n---\n";
|
||||
expect(extractTriggers(fm)).toEqual(["double quoted", "single quoted", "unquoted"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("checkResolvable — real skills directory", () => {
|
||||
const report = checkResolvable(SKILLS_DIR);
|
||||
|
||||
|
||||
@@ -8,13 +8,15 @@
|
||||
//
|
||||
// PGLite-only: in-memory engine, no DATABASE_URL needed.
|
||||
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { loadConfigWithEngine, type GBrainConfig } from '../src/core/config.ts';
|
||||
import {
|
||||
__setRerankTransportForTests,
|
||||
configureGateway,
|
||||
getEmbeddingModel,
|
||||
getMultimodalModel,
|
||||
rerank,
|
||||
resetGateway,
|
||||
} from '../src/core/ai/gateway.ts';
|
||||
import type { AIGatewayConfig } from '../src/core/ai/types.ts';
|
||||
@@ -52,10 +54,16 @@ afterAll(async () => {
|
||||
|
||||
beforeEach(async () => {
|
||||
resetGateway();
|
||||
__setRerankTransportForTests(null);
|
||||
// Clear any prior config rows so tests are independent. setConfig with
|
||||
// empty string is treated as undefined by loadConfigWithEngine (per
|
||||
// dbStr semantics), so this is safe to call between tests.
|
||||
await engine.setConfig('embedding_multimodal_model', '');
|
||||
await engine.setConfig('provider_base_urls.llama-server-reranker', '');
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
__setRerankTransportForTests(null);
|
||||
});
|
||||
|
||||
describe('cli connectEngine — embedding_multimodal_model DB→gateway plumbing', () => {
|
||||
@@ -122,4 +130,34 @@ describe('cli connectEngine — embedding_multimodal_model DB→gateway plumbing
|
||||
expect(getEmbeddingModel()).toBe('openai:text-embedding-3-large');
|
||||
expect(getMultimodalModel()).toBeUndefined();
|
||||
});
|
||||
|
||||
test('DB-set provider_base_urls.llama-server-reranker flows to gateway.rerank URL', async () => {
|
||||
await engine.setConfig('provider_base_urls.llama-server-reranker', 'http://127.0.0.1:8091/v1');
|
||||
|
||||
const baseConfig: GBrainConfig = {
|
||||
engine: 'pglite',
|
||||
embedding_model: 'openai:text-embedding-3-large',
|
||||
embedding_dimensions: 1536,
|
||||
};
|
||||
|
||||
const merged = await loadConfigWithEngine(engine, baseConfig);
|
||||
configureGateway(buildGatewayConfig(merged!));
|
||||
|
||||
let capturedUrl = '';
|
||||
__setRerankTransportForTests(async (url) => {
|
||||
capturedUrl = url;
|
||||
return new Response(JSON.stringify({ results: [{ index: 0, relevance_score: 0.9 }] }), {
|
||||
status: 200,
|
||||
headers: { 'content-type': 'application/json' },
|
||||
});
|
||||
});
|
||||
|
||||
await rerank({
|
||||
query: 'q',
|
||||
documents: ['d'],
|
||||
model: 'llama-server-reranker:qwen3-reranker-4b',
|
||||
});
|
||||
|
||||
expect(capturedUrl).toBe('http://127.0.0.1:8091/v1/rerank');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -345,3 +345,69 @@ describe('v0.41 T6: runPhaseSynthesizeConcepts via stubbed chat', () => {
|
||||
expect((page[0].fm as Record<string, unknown>).tier).toBe('T1');
|
||||
});
|
||||
});
|
||||
|
||||
// #2123 — extract_atoms must stamp `concepts` so synthesize_concepts has
|
||||
// material. The pre-fix pipeline was broken end-to-end: the extractor
|
||||
// never wrote the field, and every synthesize_concepts cycle skipped with
|
||||
// "no atoms with concept refs". The earlier describe blocks feed
|
||||
// synthesize via the `_atoms` seam, which is exactly how the gap survived
|
||||
// — so the last test here goes extractor → REAL frontmatter → real DB
|
||||
// query path → concept page.
|
||||
describe('#2123: concepts label parsing', () => {
|
||||
test('keeps valid kebab-case labels', () => {
|
||||
const raw = `[{"title":"T","atom_type":"insight","body":"b","concepts":["captive-portal","tls-certificates"]}]`;
|
||||
expect(parseAtomsResponse(raw)[0].concepts).toEqual(['captive-portal', 'tls-certificates']);
|
||||
});
|
||||
|
||||
test('filters non-kebab labels, keeps the rest', () => {
|
||||
const raw = `[{"title":"T","atom_type":"insight","body":"b","concepts":["Captive Portal","tp_link","UPPER","valid-label"]}]`;
|
||||
expect(parseAtomsResponse(raw)[0].concepts).toEqual(['valid-label']);
|
||||
});
|
||||
|
||||
test('truncates to 3 labels', () => {
|
||||
const raw = `[{"title":"T","atom_type":"insight","body":"b","concepts":["a","b","c","d","e"]}]`;
|
||||
expect(parseAtomsResponse(raw)[0].concepts).toEqual(['a', 'b', 'c']);
|
||||
});
|
||||
|
||||
test('absent / non-array / all-invalid → undefined', () => {
|
||||
expect(parseAtomsResponse(`[{"title":"T","atom_type":"insight","body":"b"}]`)[0].concepts).toBeUndefined();
|
||||
expect(parseAtomsResponse(`[{"title":"T","atom_type":"insight","body":"b","concepts":"not-an-array"}]`)[0].concepts).toBeUndefined();
|
||||
expect(parseAtomsResponse(`[{"title":"T","atom_type":"insight","body":"b","concepts":["Bad Label!"]}]`)[0].concepts).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('#2123: extractor stamps concepts → synthesize_concepts consumes via real DB path', () => {
|
||||
test('end-to-end: atoms with shared label materialize a concept page', async () => {
|
||||
const chat = stubChat(`[
|
||||
{"title":"Cert warning on guest wifi","atom_type":"insight","body":"Portal redirects to an IP-based HTTPS URL.","concepts":["captive-portal"]},
|
||||
{"title":"iPhone portal popup is flaky","atom_type":"critique","body":"CNA probe behavior differs across iOS versions.","concepts":["captive-portal"]}
|
||||
]`);
|
||||
const extract = await runPhaseExtractAtoms(engine, {
|
||||
_transcripts: [{ filePath: '/fake/notes.txt', content: 'content', contentHash: 'cc2123' }],
|
||||
_pages: [],
|
||||
_chat: chat,
|
||||
});
|
||||
expect(extract.status).toBe('ok');
|
||||
expect(extract.details?.atoms_extracted).toBe(2);
|
||||
|
||||
// Frontmatter really carries the label (a jsonb array, not a string).
|
||||
const stamped = await engine.executeRaw<{ concepts: unknown }>(
|
||||
`SELECT frontmatter->'concepts' AS concepts FROM pages WHERE type = 'atom'`,
|
||||
);
|
||||
expect(stamped.length).toBe(2);
|
||||
for (const row of stamped) {
|
||||
const arr = typeof row.concepts === 'string' ? JSON.parse(row.concepts) : row.concepts;
|
||||
expect(arr).toEqual(['captive-portal']);
|
||||
}
|
||||
|
||||
// NO _atoms seam: synthesize discovers the atoms through its own
|
||||
// DB query — this is the path that was dead before the fix.
|
||||
const synth = await runPhaseSynthesizeConcepts(engine, { _chat: stubChat('unused — T3 is deterministic') });
|
||||
expect(synth.status).toBe('ok');
|
||||
expect(synth.details?.concepts_written).toBe(1);
|
||||
const concept = await engine.executeRaw<{ slug: string }>(
|
||||
`SELECT slug FROM pages WHERE slug = 'concepts/captive-portal' AND type = 'concept'`,
|
||||
);
|
||||
expect(concept.length).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
// Regression coverage for the `gbrain doctor --remediation-plan` verdict
|
||||
// contradiction: when the brain was below target AND the target was
|
||||
// unreachable, the human renderer printed "Target unreachable: max with
|
||||
// autonomous remediation is N/100" followed immediately by "No
|
||||
// remediations needed. Brain is at target." — two consecutive lines that
|
||||
// contradicted each other and hid the real next step.
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { renderRemediationPlanLines } from '../src/commands/doctor.ts';
|
||||
|
||||
type Plan = Parameters<typeof renderRemediationPlanLines>[0];
|
||||
|
||||
function planFixture(overrides: Partial<Plan>): Plan {
|
||||
return {
|
||||
brain_score_current: 0,
|
||||
target_unreachable: false,
|
||||
max_reachable_score: 100,
|
||||
plan: [],
|
||||
est_total_seconds: 0,
|
||||
est_total_usd_cost: 0,
|
||||
blocked: [],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('renderRemediationPlanLines', () => {
|
||||
test('unreachable + brain below target — never claims "Brain is at target"', () => {
|
||||
const plan = planFixture({
|
||||
brain_score_current: 45,
|
||||
target_unreachable: true,
|
||||
max_reachable_score: 70,
|
||||
plan: [],
|
||||
blocked: [{ check: 'link_density', reason: 'no enrichment keys configured' }],
|
||||
});
|
||||
const text = renderRemediationPlanLines(plan, 90).join('\n');
|
||||
expect(text).toContain('Brain score: 45/100');
|
||||
expect(text).toContain('Target unreachable: max with autonomous remediation is 70/100');
|
||||
expect(text).not.toContain('Brain is at target');
|
||||
expect(text).toContain('Blocked checks');
|
||||
});
|
||||
|
||||
test('reachable, brain at or above target, no plan — emits the "at target" line', () => {
|
||||
const plan = planFixture({
|
||||
brain_score_current: 95,
|
||||
target_unreachable: false,
|
||||
max_reachable_score: 100,
|
||||
plan: [],
|
||||
});
|
||||
const text = renderRemediationPlanLines(plan, 90).join('\n');
|
||||
expect(text).toContain('Brain is at target');
|
||||
expect(text).not.toContain('Target unreachable');
|
||||
});
|
||||
|
||||
test('brain at exact target with empty plan — still "at target"', () => {
|
||||
const plan = planFixture({
|
||||
brain_score_current: 90,
|
||||
target_unreachable: false,
|
||||
plan: [],
|
||||
});
|
||||
const text = renderRemediationPlanLines(plan, 90).join('\n');
|
||||
expect(text).toContain('Brain is at target');
|
||||
});
|
||||
|
||||
test('brain below target with plan steps — lists the plan, no "at target" line', () => {
|
||||
const plan = planFixture({
|
||||
brain_score_current: 60,
|
||||
target_unreachable: false,
|
||||
max_reachable_score: 100,
|
||||
est_total_seconds: 120,
|
||||
est_total_usd_cost: 0.4,
|
||||
plan: [
|
||||
{ step: 1, severity: 'high', job: 'embed-coverage', rationale: 'missing embeddings' },
|
||||
{ step: 2, severity: 'med', job: 'consolidate', rationale: 'pending entity merges', est_usd_cost: 0.4 },
|
||||
],
|
||||
});
|
||||
const lines = renderRemediationPlanLines(plan, 90);
|
||||
const text = lines.join('\n');
|
||||
expect(text).toContain('Plan: 2 step(s)');
|
||||
expect(text).toContain('1. [high] embed-coverage');
|
||||
expect(text).toContain('2. [med] consolidate');
|
||||
expect(text).toContain('($0.40)');
|
||||
expect(text).not.toContain('Brain is at target');
|
||||
});
|
||||
|
||||
test('unreachable but a partial plan exists — plan prints, "at target" suppressed', () => {
|
||||
const plan = planFixture({
|
||||
brain_score_current: 30,
|
||||
target_unreachable: true,
|
||||
max_reachable_score: 55,
|
||||
est_total_seconds: 90,
|
||||
est_total_usd_cost: 0.2,
|
||||
plan: [
|
||||
{ step: 1, severity: 'high', job: 'embed-coverage', rationale: 'reach max_reachable' },
|
||||
],
|
||||
blocked: [{ check: 'enrichment', reason: 'no provider key configured' }],
|
||||
});
|
||||
const text = renderRemediationPlanLines(plan, 90).join('\n');
|
||||
expect(text).toContain('Target unreachable: max with autonomous remediation is 55/100');
|
||||
expect(text).toContain('Plan: 1 step(s)');
|
||||
expect(text).toContain('Blocked checks');
|
||||
expect(text).not.toContain('Brain is at target');
|
||||
});
|
||||
});
|
||||
@@ -20,6 +20,10 @@ import {
|
||||
const skip = !hasDatabase();
|
||||
const describeE2E = skip ? describe.skip : describe;
|
||||
|
||||
if (skip) {
|
||||
console.log('Skipping E2E doctor --progress-json tests (DATABASE_URL not set)');
|
||||
}
|
||||
|
||||
const CLI = join(import.meta.dir, '..', '..', 'src', 'cli.ts');
|
||||
|
||||
describeE2E('gbrain doctor --progress-json (E2E)', () => {
|
||||
|
||||
@@ -29,9 +29,15 @@ afterAll(async () => {
|
||||
});
|
||||
|
||||
async function truncateAll() {
|
||||
for (const t of ['content_chunks', 'links', 'tags', 'raw_data', 'timeline_entries', 'page_versions', 'ingest_log', 'pages']) {
|
||||
for (const t of ['content_chunks', 'links', 'tags', 'raw_data', 'timeline_entries', 'page_versions', 'ingest_log', 'config', 'pages']) {
|
||||
await (engine as any).db.exec(`DELETE FROM ${t}`);
|
||||
}
|
||||
// Re-seed the two config keys this file touches back to their documented
|
||||
// defaults (both default to ON). This makes every test deterministic even if
|
||||
// an earlier test threw before its finally restored auto_link/auto_timeline,
|
||||
// and even though absent-key already resolves truthy via isAuto*Enabled.
|
||||
await engine.setConfig('auto_link', 'true');
|
||||
await engine.setConfig('auto_timeline', 'true');
|
||||
}
|
||||
|
||||
function makeContext(): OperationContext {
|
||||
@@ -77,10 +83,12 @@ describe('E2E graph quality (v0.10.1 pipeline)', () => {
|
||||
await runExtract(engine, ['links', '--source', 'db']);
|
||||
await runExtract(engine, ['timeline', '--source', 'db']);
|
||||
|
||||
// Verify graph populated.
|
||||
// Verify graph populated. Concrete floors derived from the seeded fixtures:
|
||||
// resolvable entity refs: alice->acme, bob->acme, standup->alice, standup->bob = 4
|
||||
// timeline lines: alice(2) + bob(1) + acme(1) + standup(1) = 5
|
||||
const stats = await engine.getStats();
|
||||
expect(stats.link_count).toBeGreaterThan(0);
|
||||
expect(stats.timeline_entry_count).toBeGreaterThan(0);
|
||||
expect(stats.link_count).toBeGreaterThanOrEqual(4);
|
||||
expect(stats.timeline_entry_count).toBeGreaterThanOrEqual(5);
|
||||
|
||||
// Verify typed link inference.
|
||||
const aliceLinks = await engine.getLinks('people/alice');
|
||||
@@ -91,7 +99,16 @@ describe('E2E graph quality (v0.10.1 pipeline)', () => {
|
||||
const bobAcme = bobLinks.find(l => l.to_slug === 'companies/acme');
|
||||
expect(bobAcme?.link_type).toBe('invested_in');
|
||||
|
||||
// The standup meeting references both Alice and Bob as attendees. Assert the
|
||||
// exact attendee edges are present and typed 'attended' (a plain .every()
|
||||
// would silently pass if a meeting->company edge were misclassified or if the
|
||||
// attendee edges were missing entirely).
|
||||
const meetingLinks = await engine.getLinks('meetings/standup');
|
||||
const attended = new Set(
|
||||
meetingLinks.filter(l => l.link_type === 'attended').map(l => l.to_slug),
|
||||
);
|
||||
expect(attended.has('people/alice')).toBe(true);
|
||||
expect(attended.has('people/bob')).toBe(true);
|
||||
expect(meetingLinks.every(l => l.link_type === 'attended')).toBe(true);
|
||||
});
|
||||
|
||||
@@ -118,7 +135,9 @@ Attendees: [Alice](people/alice). Discussed [Acme](companies/acme).
|
||||
// The response should include auto_links results.
|
||||
expect((result as any).auto_links).toBeDefined();
|
||||
const autoLinks = (result as any).auto_links;
|
||||
expect(autoLinks.created).toBeGreaterThan(0);
|
||||
// The page references exactly two seeded, resolvable targets (Alice + Acme),
|
||||
// so exactly two links are created.
|
||||
expect(autoLinks.created).toBe(2);
|
||||
expect(autoLinks.errors).toBe(0);
|
||||
|
||||
// Verify links actually exist in DB.
|
||||
@@ -283,6 +302,53 @@ Mention of [Alice](people/alice).
|
||||
expect(paths[0].link_type).toBe('works_at');
|
||||
});
|
||||
|
||||
test('graph-query traversal: direction out and both, plus depth:2 multi-hop', async () => {
|
||||
// Seed a 2-hop chain: alice -works_at-> acme -partnered_with-> beta.
|
||||
await engine.putPage('people/alice', { type: 'person', title: 'Alice', compiled_truth: '', timeline: '' });
|
||||
await engine.putPage('companies/acme', { type: 'company', title: 'Acme', compiled_truth: '', timeline: '' });
|
||||
await engine.putPage('companies/beta', { type: 'company', title: 'Beta', compiled_truth: '', timeline: '' });
|
||||
await engine.addLink('people/alice', 'companies/acme', '', 'works_at');
|
||||
await engine.addLink('companies/acme', 'companies/beta', '', 'partnered_with');
|
||||
|
||||
// direction:'out' from alice, depth 1 -> only the first hop.
|
||||
const out1 = await engine.traversePaths('people/alice', { direction: 'out', depth: 1 });
|
||||
expect(out1.length).toBe(1);
|
||||
expect(out1[0].from_slug).toBe('people/alice');
|
||||
expect(out1[0].to_slug).toBe('companies/acme');
|
||||
expect(out1[0].depth).toBe(1);
|
||||
|
||||
// depth:2 -> both hops, depths 1 and 2.
|
||||
const out2 = await engine.traversePaths('people/alice', { direction: 'out', depth: 2 });
|
||||
const out2Edges = new Set(out2.map(p => `${p.from_slug}->${p.to_slug}@${p.depth}`));
|
||||
expect(out2Edges.has('people/alice->companies/acme@1')).toBe(true);
|
||||
expect(out2Edges.has('companies/acme->companies/beta@2')).toBe(true);
|
||||
expect(out2.length).toBe(2);
|
||||
|
||||
// direction:'both' from acme depth 1 -> sees the inbound edge from alice AND
|
||||
// the outbound edge to beta. Edges keep their natural from->to orientation.
|
||||
const both = await engine.traversePaths('companies/acme', { direction: 'both', depth: 1 });
|
||||
const bothEdges = new Set(both.map(p => `${p.from_slug}->${p.to_slug}`));
|
||||
expect(bothEdges.has('people/alice->companies/acme')).toBe(true);
|
||||
expect(bothEdges.has('companies/acme->companies/beta')).toBe(true);
|
||||
});
|
||||
|
||||
test('graph-query cycle safety: A->B->A terminates and returns bounded results', async () => {
|
||||
await engine.putPage('people/alice', { type: 'person', title: 'Alice', compiled_truth: '', timeline: '' });
|
||||
await engine.putPage('people/bob', { type: 'person', title: 'Bob', compiled_truth: '', timeline: '' });
|
||||
// Create a 2-cycle: alice -> bob -> alice.
|
||||
await engine.addLink('people/alice', 'people/bob', '', 'knows');
|
||||
await engine.addLink('people/bob', 'people/alice', '', 'knows');
|
||||
|
||||
// High depth must NOT loop forever; the visited-set guard bounds the walk.
|
||||
const paths = await engine.traversePaths('people/alice', { direction: 'out', depth: 100 });
|
||||
const edges = new Set(paths.map(p => `${p.from_slug}->${p.to_slug}`));
|
||||
// Both edges of the cycle are reachable exactly once.
|
||||
expect(edges.has('people/alice->people/bob')).toBe(true);
|
||||
expect(edges.has('people/bob->people/alice')).toBe(true);
|
||||
// Bounded: there are only two edges in the graph, so no path explosion.
|
||||
expect(paths.length).toBe(2);
|
||||
});
|
||||
|
||||
test('search backlink boost: well-connected pages rank higher', async () => {
|
||||
// Create 3 pages all matching a search term, but with different inbound link counts.
|
||||
await engine.putPage('topic/popular', {
|
||||
|
||||
@@ -21,6 +21,10 @@ import { hasDatabase, setupDB, teardownDB, getEngine, getConn } from './helpers.
|
||||
const skip = !hasDatabase();
|
||||
const describeE2E = skip ? describe.skip : describe;
|
||||
|
||||
if (skip) {
|
||||
console.log('Skipping E2E JSONB roundtrip tests (DATABASE_URL not set)');
|
||||
}
|
||||
|
||||
describeE2E('E2E: JSONB roundtrip — v0.12.1 reliability wave', () => {
|
||||
beforeAll(async () => { await setupDB(); });
|
||||
afterAll(async () => { await teardownDB(); });
|
||||
|
||||
@@ -56,6 +56,7 @@ describe('E2E: MCP Tool Generation', () => {
|
||||
expect(names).toContain('get_health');
|
||||
expect(names).toContain('sync_brain');
|
||||
expect(names).toContain('file_upload');
|
||||
expect(names).toContain('find_orphans');
|
||||
});
|
||||
|
||||
test('MCP server module can be imported', async () => {
|
||||
|
||||
@@ -175,6 +175,15 @@ describeE2E('E2E: Search', () => {
|
||||
for (const [query, score] of Object.entries(scores)) {
|
||||
console.log(` "${query}": ${(score * 100).toFixed(0)}%`);
|
||||
}
|
||||
|
||||
// Guard value: every known-item query must surface at least one ground-truth
|
||||
// doc in the top 5. This is a deliberately loose floor (not a tuned P@5
|
||||
// threshold) — it catches a total keyword-retrieval regression without
|
||||
// breaking on every scoring/fixture tweak. Without it this test asserted
|
||||
// nothing and a 0%-precision result passed silently.
|
||||
for (const [query, score] of Object.entries(scores)) {
|
||||
expect(score).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -205,10 +214,22 @@ describeE2E('E2E: Links', () => {
|
||||
}, 30_000);
|
||||
|
||||
test('traverse_graph finds connected pages', async () => {
|
||||
// Links should already be added from prior test in this describe block
|
||||
const graph = await callOp('traverse_graph', { slug: 'people/sarah-chen', depth: 2 }) as any;
|
||||
// Self-contained: do not depend on a prior test's add_link. add_link is
|
||||
// idempotent (ON CONFLICT DO NOTHING), so re-adding here is safe whether or
|
||||
// not the round-trip test ran first, and the test no longer false-passes or
|
||||
// false-fails based on describe-block ordering.
|
||||
await callOp('add_link', {
|
||||
from: 'people/sarah-chen',
|
||||
to: 'companies/novamind',
|
||||
link_type: 'founded',
|
||||
});
|
||||
|
||||
const graph = await callOp('traverse_graph', { slug: 'people/sarah-chen', depth: 2 }) as any[];
|
||||
expect(Array.isArray(graph)).toBe(true);
|
||||
expect(graph.length).toBeGreaterThanOrEqual(1);
|
||||
// Content assertion, not just shape: the linked company must be reachable.
|
||||
const reachable = graph.map((n: any) => n.slug ?? n.to_slug ?? n.to_page_slug);
|
||||
expect(reachable).toContain('companies/novamind');
|
||||
});
|
||||
|
||||
test('remove_link removes the link', async () => {
|
||||
@@ -469,8 +490,14 @@ describeE2E('E2E: Admin', () => {
|
||||
test('get_health returns valid structure', async () => {
|
||||
const health = await callOp('get_health') as any;
|
||||
expect(health).toBeDefined();
|
||||
expect(typeof health.page_count).toBe('number');
|
||||
expect(typeof health.embed_coverage).toBe('number');
|
||||
// Value bounds, not just types: page_count must match the fixture inventory
|
||||
// and embed_coverage is a 0..1 fraction (src/commands/doctor.ts multiplies
|
||||
// by 100 and compares to 0.9). Type-only checks let embed_coverage: -9999
|
||||
// through; these catch a genuinely broken health payload.
|
||||
expect(health.page_count).toBe(16);
|
||||
expect(Number.isFinite(health.embed_coverage)).toBe(true);
|
||||
expect(health.embed_coverage).toBeGreaterThanOrEqual(0);
|
||||
expect(health.embed_coverage).toBeLessThanOrEqual(1);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -488,7 +515,17 @@ describeE2E('E2E: Chunks & Resolution', () => {
|
||||
test('get_chunks returns chunks for imported page', async () => {
|
||||
const chunks = await callOp('get_chunks', { slug: 'people/sarah-chen' }) as any[];
|
||||
expect(chunks.length).toBeGreaterThan(0);
|
||||
expect(chunks[0].chunk_text).toBeTruthy();
|
||||
// Content + ordering, not just truthiness (a whitespace-only chunk is truthy):
|
||||
// every chunk has real text and a numeric index, the indexes are
|
||||
// non-decreasing in return order, and the page's own name appears somewhere.
|
||||
for (const c of chunks) {
|
||||
expect(typeof c.chunk_text).toBe('string');
|
||||
expect(c.chunk_text.trim().length).toBeGreaterThan(0);
|
||||
expect(typeof c.chunk_index).toBe('number');
|
||||
}
|
||||
const indexes = chunks.map((c: any) => c.chunk_index);
|
||||
expect(indexes).toEqual([...indexes].sort((x, y) => x - y));
|
||||
expect(chunks.some((c: any) => c.chunk_text.includes('Sarah'))).toBe(true);
|
||||
}, 30_000);
|
||||
|
||||
test('resolve_slugs finds partial match', async () => {
|
||||
@@ -662,9 +699,29 @@ describeE2E('E2E: file_list LIMIT enforcement', () => {
|
||||
}, 30_000);
|
||||
|
||||
test('file_list without slug also respects LIMIT 100', async () => {
|
||||
// The 150 rows from the previous test are still in the DB
|
||||
// Self-sufficient: seed our own >100 rows rather than relying on the
|
||||
// previous test's 150 rows surviving in the DB. A bun reorder, a focused
|
||||
// `-t` run, or a failure mid-insert in the prior test would otherwise leave
|
||||
// this asserting against an indeterminate row count.
|
||||
const sql = getConn();
|
||||
const seedSlug = 'test-limit-noslug';
|
||||
await sql`
|
||||
INSERT INTO pages (slug, title, type, compiled_truth, frontmatter)
|
||||
VALUES (${seedSlug}, ${'Test Limit NoSlug'}, ${'note'}, ${'body'}, ${'{}'}::jsonb)
|
||||
ON CONFLICT (source_id, slug) DO NOTHING
|
||||
`;
|
||||
for (let i = 0; i < 120; i++) {
|
||||
await sql`
|
||||
INSERT INTO files (page_slug, filename, storage_path, mime_type, size_bytes, content_hash, metadata)
|
||||
VALUES (${seedSlug}, ${'nf-' + String(i).padStart(3, '0') + '.txt'}, ${seedSlug + '/nf-' + i + '.txt'}, ${'text/plain'}, ${100}, ${'nhash-' + i}, ${'{}'}::jsonb)
|
||||
ON CONFLICT (storage_path) DO NOTHING
|
||||
`;
|
||||
}
|
||||
const total = await sql`SELECT count(*)::int AS n FROM files`;
|
||||
expect(Number(total[0].n)).toBeGreaterThan(100); // cap is actually exercised
|
||||
|
||||
const files = await callOp('file_list', {}) as any[];
|
||||
expect(files.length).toBeLessThanOrEqual(100);
|
||||
expect(files.length).toBe(100);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
+149
-111
@@ -78,6 +78,19 @@ function freshTempHome(label: string) {
|
||||
return dir;
|
||||
}
|
||||
|
||||
// Restore HOME/PATH to the captured originals. Called from each test's
|
||||
// `finally` so a throw mid-test can never leave HOME/PATH pointed at a temp
|
||||
// dir for the rest of the bun process (which would silently break unrelated
|
||||
// suites that read HOME). PATH keeps the shim prepended because the
|
||||
// module-level shim install is what subsequent tests in this suite rely on;
|
||||
// afterAll does the final teardown to the pristine origPath.
|
||||
function restoreHomePath() {
|
||||
if (origHome === undefined) delete process.env.HOME;
|
||||
else process.env.HOME = origHome;
|
||||
if (origPath === undefined) delete process.env.PATH;
|
||||
else process.env.PATH = `${fakeBinDir}:${origPath ?? ''}`;
|
||||
}
|
||||
|
||||
beforeAll(() => {
|
||||
if (SKIP) {
|
||||
console.log('[migration-flow.e2e] DATABASE_URL not set — skipping.');
|
||||
@@ -100,6 +113,15 @@ afterAll(() => {
|
||||
|
||||
beforeEach(() => {
|
||||
if (SKIP) return;
|
||||
// Robust restore: if a prior test threw before its own finally ran (or
|
||||
// before afterAll), HOME/PATH could still point at a dead temp dir. Reset
|
||||
// them to the captured originals at the start of every test so a throw in
|
||||
// one test can never leak a temp HOME/PATH into sibling suites that read
|
||||
// them. freshTempHome() re-points HOME per test immediately after this.
|
||||
if (origHome === undefined) delete process.env.HOME;
|
||||
else process.env.HOME = origHome;
|
||||
if (origPath === undefined) delete process.env.PATH;
|
||||
else process.env.PATH = `${fakeBinDir}:${origPath ?? ''}`;
|
||||
try { if (tmp) rmSync(tmp, { recursive: true, force: true }); } catch { /* best-effort */ }
|
||||
});
|
||||
|
||||
@@ -114,144 +136,160 @@ const COMMON_OPTS = {
|
||||
describeE2E('E2E: v0.11.0 orchestrator against live Postgres', () => {
|
||||
test('fresh install flow: schema → smoke → prefs → host-rewrite → completed', async () => {
|
||||
tmp = freshTempHome('fresh');
|
||||
const result = await v0_11_0.orchestrator(COMMON_OPTS);
|
||||
try {
|
||||
const result = await v0_11_0.orchestrator(COMMON_OPTS);
|
||||
|
||||
// Orchestrator returns a structured result (status is `complete` when
|
||||
// no pending-host-work TODOs fired, `partial` otherwise).
|
||||
expect(result.version).toBe('0.11.0');
|
||||
expect(['complete', 'partial']).toContain(result.status);
|
||||
// Orchestrator returns a structured result (status is `complete` when
|
||||
// no pending-host-work TODOs fired, `partial` otherwise).
|
||||
expect(result.version).toBe('0.11.0');
|
||||
expect(['complete', 'partial']).toContain(result.status);
|
||||
|
||||
// Phase D: preferences.json exists with 0o600 + mode=pain_triggered.
|
||||
const prefsPath = join(tmp, '.gbrain', 'preferences.json');
|
||||
expect(existsSync(prefsPath)).toBe(true);
|
||||
expect(statSync(prefsPath).mode & 0o777).toBe(0o600);
|
||||
const prefs = loadPreferences();
|
||||
expect(prefs.minion_mode).toBe('pain_triggered');
|
||||
expect(prefs.set_at).toBeTruthy();
|
||||
expect(prefs.set_in_version).toBeTruthy();
|
||||
// Phase D: preferences.json exists with 0o600 + mode=pain_triggered.
|
||||
const prefsPath = join(tmp, '.gbrain', 'preferences.json');
|
||||
expect(existsSync(prefsPath)).toBe(true);
|
||||
expect(statSync(prefsPath).mode & 0o777).toBe(0o600);
|
||||
const prefs = loadPreferences();
|
||||
expect(prefs.minion_mode).toBe('pain_triggered');
|
||||
expect(prefs.set_at).toBeTruthy();
|
||||
expect(prefs.set_in_version).toBeTruthy();
|
||||
|
||||
// Bug 3 (v0.14.2) — orchestrator no longer writes completed.jsonl.
|
||||
// The runner (apply-migrations.ts) persists the result after the
|
||||
// orchestrator returns. A direct orchestrator call in E2E leaves the
|
||||
// ledger empty; the runner path is tested separately in
|
||||
// test/apply-migrations.test.ts + test/migration-resume.test.ts.
|
||||
const completed = loadCompletedMigrations();
|
||||
const v0110Entries = completed.filter(e => e.version === '0.11.0');
|
||||
expect(v0110Entries.length).toBe(0);
|
||||
// Bug 3 (v0.14.2) — orchestrator no longer writes completed.jsonl.
|
||||
// The runner (apply-migrations.ts) persists the result after the
|
||||
// orchestrator returns. A direct orchestrator call in E2E leaves the
|
||||
// ledger empty; the runner path is tested separately in
|
||||
// test/apply-migrations.test.ts + test/migration-resume.test.ts.
|
||||
const completed = loadCompletedMigrations();
|
||||
const v0110Entries = completed.filter(e => e.version === '0.11.0');
|
||||
expect(v0110Entries.length).toBe(0);
|
||||
|
||||
// Phase F is skipped per COMMON_OPTS — autopilot should NOT have been
|
||||
// installed on this host.
|
||||
expect(result.autopilot_installed).toBe(false);
|
||||
// Phase F is skipped per COMMON_OPTS — autopilot should NOT have been
|
||||
// installed on this host.
|
||||
expect(result.autopilot_installed).toBe(false);
|
||||
} finally {
|
||||
restoreHomePath();
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
test('idempotent rerun: second invocation is a safe no-op', async () => {
|
||||
tmp = freshTempHome('rerun');
|
||||
const first = await v0_11_0.orchestrator(COMMON_OPTS);
|
||||
expect(['complete', 'partial']).toContain(first.status);
|
||||
try {
|
||||
const first = await v0_11_0.orchestrator(COMMON_OPTS);
|
||||
expect(['complete', 'partial']).toContain(first.status);
|
||||
|
||||
const second = await v0_11_0.orchestrator(COMMON_OPTS);
|
||||
expect(['complete', 'partial']).toContain(second.status);
|
||||
const second = await v0_11_0.orchestrator(COMMON_OPTS);
|
||||
expect(['complete', 'partial']).toContain(second.status);
|
||||
|
||||
// Bug 3 (v0.14.2) — orchestrator does not write completed.jsonl, so
|
||||
// repeated direct invocations don't accumulate ledger entries. Assert
|
||||
// the preferences state stays stable (the real idempotency signal for
|
||||
// this orchestrator is "running again doesn't corrupt preferences").
|
||||
expect(loadPreferences().minion_mode).toBe('pain_triggered');
|
||||
const completed = loadCompletedMigrations();
|
||||
expect(completed.filter(e => e.version === '0.11.0').length).toBe(0);
|
||||
// Bug 3 (v0.14.2) — orchestrator does not write completed.jsonl, so
|
||||
// repeated direct invocations don't accumulate ledger entries. Assert
|
||||
// the preferences state stays stable (the real idempotency signal for
|
||||
// this orchestrator is "running again doesn't corrupt preferences").
|
||||
expect(loadPreferences().minion_mode).toBe('pain_triggered');
|
||||
const completed = loadCompletedMigrations();
|
||||
expect(completed.filter(e => e.version === '0.11.0').length).toBe(0);
|
||||
} finally {
|
||||
restoreHomePath();
|
||||
}
|
||||
}, 90_000);
|
||||
|
||||
test('host rewrite: builtin handlers auto-rewritten, non-builtins queued as JSONL TODOs', async () => {
|
||||
tmp = freshTempHome('host-rewrite');
|
||||
// Fixture: AGENTS.md + cron/jobs.json with a mix of gbrain-builtin and
|
||||
// non-builtin handlers.
|
||||
const claudeDir = join(tmp, '.claude');
|
||||
mkdirSync(claudeDir, { recursive: true });
|
||||
writeFileSync(
|
||||
join(claudeDir, 'AGENTS.md'),
|
||||
'# Test AGENTS.md\n\nSome existing content referencing sessions_spawn routing.\n',
|
||||
);
|
||||
mkdirSync(join(claudeDir, 'cron'), { recursive: true });
|
||||
writeFileSync(
|
||||
join(claudeDir, 'cron', 'jobs.json'),
|
||||
JSON.stringify({
|
||||
jobs: [
|
||||
{ schedule: '*/5 * * * *', kind: 'agentTurn', skill: 'sync' }, // builtin
|
||||
{ schedule: '0 */30 * * *', kind: 'agentTurn', skill: 'ea-inbox-sweep' }, // non-builtin
|
||||
{ schedule: '*/10 * * * *', kind: 'agentTurn', skill: 'embed' }, // builtin
|
||||
{ schedule: '0 8 * * *', kind: 'agentTurn', skill: 'morning-briefing' }, // non-builtin
|
||||
],
|
||||
}, null, 2) + '\n',
|
||||
);
|
||||
try {
|
||||
// Fixture: AGENTS.md + cron/jobs.json with a mix of gbrain-builtin and
|
||||
// non-builtin handlers.
|
||||
const claudeDir = join(tmp, '.claude');
|
||||
mkdirSync(claudeDir, { recursive: true });
|
||||
writeFileSync(
|
||||
join(claudeDir, 'AGENTS.md'),
|
||||
'# Test AGENTS.md\n\nSome existing content referencing sessions_spawn routing.\n',
|
||||
);
|
||||
mkdirSync(join(claudeDir, 'cron'), { recursive: true });
|
||||
writeFileSync(
|
||||
join(claudeDir, 'cron', 'jobs.json'),
|
||||
JSON.stringify({
|
||||
jobs: [
|
||||
{ schedule: '*/5 * * * *', kind: 'agentTurn', skill: 'sync' }, // builtin
|
||||
{ schedule: '0 */30 * * *', kind: 'agentTurn', skill: 'ea-inbox-sweep' }, // non-builtin
|
||||
{ schedule: '*/10 * * * *', kind: 'agentTurn', skill: 'embed' }, // builtin
|
||||
{ schedule: '0 8 * * *', kind: 'agentTurn', skill: 'morning-briefing' }, // non-builtin
|
||||
],
|
||||
}, null, 2) + '\n',
|
||||
);
|
||||
|
||||
const result = await v0_11_0.orchestrator(COMMON_OPTS);
|
||||
const result = await v0_11_0.orchestrator(COMMON_OPTS);
|
||||
|
||||
// Builtins rewritten in place; non-builtins left alone.
|
||||
const cronAfter = JSON.parse(readFileSync(join(claudeDir, 'cron', 'jobs.json'), 'utf-8'));
|
||||
expect(cronAfter.jobs[0].kind).toBe('shell'); // sync (builtin)
|
||||
expect(cronAfter.jobs[0].cmd).toContain('gbrain jobs submit sync');
|
||||
expect(cronAfter.jobs[1].kind).toBe('agentTurn'); // ea-inbox-sweep (non-builtin)
|
||||
expect(cronAfter.jobs[2].kind).toBe('shell'); // embed (builtin)
|
||||
expect(cronAfter.jobs[3].kind).toBe('agentTurn'); // morning-briefing (non-builtin)
|
||||
// Builtins rewritten in place; non-builtins left alone.
|
||||
const cronAfter = JSON.parse(readFileSync(join(claudeDir, 'cron', 'jobs.json'), 'utf-8'));
|
||||
expect(cronAfter.jobs[0].kind).toBe('shell'); // sync (builtin)
|
||||
expect(cronAfter.jobs[0].cmd).toContain('gbrain jobs submit sync');
|
||||
expect(cronAfter.jobs[1].kind).toBe('agentTurn'); // ea-inbox-sweep (non-builtin)
|
||||
expect(cronAfter.jobs[2].kind).toBe('shell'); // embed (builtin)
|
||||
expect(cronAfter.jobs[3].kind).toBe('agentTurn'); // morning-briefing (non-builtin)
|
||||
|
||||
// files_rewritten counts the 2 builtin rewrites.
|
||||
expect(result.files_rewritten).toBeGreaterThanOrEqual(2);
|
||||
// files_rewritten counts the 2 builtin rewrites.
|
||||
expect(result.files_rewritten).toBeGreaterThanOrEqual(2);
|
||||
|
||||
// pending_host_work counts the 2 non-builtin TODOs.
|
||||
expect(result.pending_host_work).toBe(2);
|
||||
// pending_host_work counts the 2 non-builtin TODOs.
|
||||
expect(result.pending_host_work).toBe(2);
|
||||
|
||||
// Status is "partial" because non-builtin TODOs remain.
|
||||
expect(result.status).toBe('partial');
|
||||
// Status is "partial" because non-builtin TODOs remain.
|
||||
expect(result.status).toBe('partial');
|
||||
|
||||
// AGENTS.md got the marker injected.
|
||||
const agentsMdAfter = readFileSync(join(claudeDir, 'AGENTS.md'), 'utf-8');
|
||||
expect(agentsMdAfter).toContain('gbrain:subagent-routing v0.11.0');
|
||||
expect(agentsMdAfter).toContain('skills/conventions/subagent-routing.md');
|
||||
// AGENTS.md got the marker injected.
|
||||
const agentsMdAfter = readFileSync(join(claudeDir, 'AGENTS.md'), 'utf-8');
|
||||
expect(agentsMdAfter).toContain('gbrain:subagent-routing v0.11.0');
|
||||
expect(agentsMdAfter).toContain('skills/conventions/subagent-routing.md');
|
||||
|
||||
// JSONL TODO file written under ~/.gbrain/migrations/.
|
||||
const jsonlPath = join(tmp, '.gbrain', 'migrations', 'pending-host-work.jsonl');
|
||||
expect(existsSync(jsonlPath)).toBe(true);
|
||||
const lines = readFileSync(jsonlPath, 'utf-8').split('\n').filter(l => l.trim());
|
||||
expect(lines.length).toBe(2);
|
||||
const todos = lines.map(l => JSON.parse(l));
|
||||
const handlers = todos.map(t => t.handler).sort();
|
||||
expect(handlers).toEqual(['ea-inbox-sweep', 'morning-briefing']);
|
||||
for (const todo of todos) {
|
||||
expect(todo.type).toBe('cron-handler-needs-host-registration');
|
||||
expect(todo.status).toBe('pending');
|
||||
expect(todo.manifest_path).toContain('cron/jobs.json');
|
||||
// JSONL TODO file written under ~/.gbrain/migrations/.
|
||||
const jsonlPath = join(tmp, '.gbrain', 'migrations', 'pending-host-work.jsonl');
|
||||
expect(existsSync(jsonlPath)).toBe(true);
|
||||
const lines = readFileSync(jsonlPath, 'utf-8').split('\n').filter(l => l.trim());
|
||||
expect(lines.length).toBe(2);
|
||||
const todos = lines.map(l => JSON.parse(l));
|
||||
const handlers = todos.map(t => t.handler).sort();
|
||||
expect(handlers).toEqual(['ea-inbox-sweep', 'morning-briefing']);
|
||||
for (const todo of todos) {
|
||||
expect(todo.type).toBe('cron-handler-needs-host-registration');
|
||||
expect(todo.status).toBe('pending');
|
||||
expect(todo.manifest_path).toContain('cron/jobs.json');
|
||||
}
|
||||
} finally {
|
||||
restoreHomePath();
|
||||
}
|
||||
}, 90_000);
|
||||
|
||||
test('resumable: partial run → orchestrator re-run → complete', async () => {
|
||||
tmp = freshTempHome('resumable');
|
||||
// Simulate a stopgap-written partial entry BEFORE running the orchestrator.
|
||||
mkdirSync(join(tmp, '.gbrain', 'migrations'), { recursive: true });
|
||||
writeFileSync(
|
||||
join(tmp, '.gbrain', 'migrations', 'completed.jsonl'),
|
||||
JSON.stringify({
|
||||
version: '0.11.0',
|
||||
status: 'partial',
|
||||
apply_migrations_pending: true,
|
||||
mode: 'pain_triggered',
|
||||
source: 'fix-v0.11.0.sh',
|
||||
ts: new Date().toISOString(),
|
||||
}) + '\n',
|
||||
);
|
||||
try {
|
||||
// Simulate a stopgap-written partial entry BEFORE running the orchestrator.
|
||||
mkdirSync(join(tmp, '.gbrain', 'migrations'), { recursive: true });
|
||||
writeFileSync(
|
||||
join(tmp, '.gbrain', 'migrations', 'completed.jsonl'),
|
||||
JSON.stringify({
|
||||
version: '0.11.0',
|
||||
status: 'partial',
|
||||
apply_migrations_pending: true,
|
||||
mode: 'pain_triggered',
|
||||
source: 'fix-v0.11.0.sh',
|
||||
ts: new Date().toISOString(),
|
||||
}) + '\n',
|
||||
);
|
||||
|
||||
// Orchestrator re-running on a partial → should succeed (schema apply
|
||||
// and smoke are idempotent; prefs are preserved from the partial
|
||||
// record; host-rewrite runs its safe-skip pass). Per Bug 3 (v0.14.2),
|
||||
// the orchestrator itself doesn't append to completed.jsonl — the
|
||||
// runner does. The stopgap's partial entry stays unchanged here.
|
||||
const result = await v0_11_0.orchestrator(COMMON_OPTS);
|
||||
expect(['complete', 'partial']).toContain(result.status);
|
||||
// Orchestrator re-running on a partial → should succeed (schema apply
|
||||
// and smoke are idempotent; prefs are preserved from the partial
|
||||
// record; host-rewrite runs its safe-skip pass). Per Bug 3 (v0.14.2),
|
||||
// the orchestrator itself doesn't append to completed.jsonl — the
|
||||
// runner does. The stopgap's partial entry stays unchanged here.
|
||||
const result = await v0_11_0.orchestrator(COMMON_OPTS);
|
||||
expect(['complete', 'partial']).toContain(result.status);
|
||||
|
||||
const completed = loadCompletedMigrations();
|
||||
const v0110 = completed.filter(e => e.version === '0.11.0');
|
||||
// Just the stopgap partial — orchestrator doesn't add its own entry.
|
||||
expect(v0110.length).toBe(1);
|
||||
expect(v0110[0].status).toBe('partial');
|
||||
expect(v0110[0].source).toBe('fix-v0.11.0.sh');
|
||||
const completed = loadCompletedMigrations();
|
||||
const v0110 = completed.filter(e => e.version === '0.11.0');
|
||||
// Just the stopgap partial — orchestrator doesn't add its own entry.
|
||||
expect(v0110.length).toBe(1);
|
||||
expect(v0110[0].status).toBe('partial');
|
||||
expect(v0110[0].source).toBe('fix-v0.11.0.sh');
|
||||
} finally {
|
||||
restoreHomePath();
|
||||
}
|
||||
}, 90_000);
|
||||
});
|
||||
|
||||
@@ -94,7 +94,7 @@ describeE2E('E2E: Minions resilience (OpenClaw real-world patterns)', () => {
|
||||
}, 30_000);
|
||||
|
||||
// --- 2. Runaway handler: ignores AbortSignal, dead-lettered by handleTimeouts ---
|
||||
test('runaway handler: ignores AbortSignal, handleTimeouts dead-letters in <2s', async () => {
|
||||
test('runaway handler: ignores AbortSignal, handleTimeouts dead-letters it', async () => {
|
||||
const { a, b } = await makeEngines();
|
||||
try {
|
||||
const queue = new MinionQueue(a);
|
||||
@@ -133,8 +133,14 @@ describeE2E('E2E: Minions resilience (OpenClaw real-world patterns)', () => {
|
||||
worker.stop();
|
||||
await startP;
|
||||
|
||||
// Correctness gate: the job MUST be dead-lettered with the timeout reason.
|
||||
// We intentionally do NOT assert a wall-clock upper bound (deadAt - started):
|
||||
// on a loaded CI runner the stall/timeout sweep cadence varies, and the only
|
||||
// thing that matters is that the runaway job terminates as 'dead'. The 3s poll
|
||||
// deadline above is the real timeout — if the sweep is too slow, finalStatus
|
||||
// stays '' and this toBe('dead') fails loudly.
|
||||
expect(finalStatus).toBe('dead');
|
||||
expect(deadAt - started).toBeLessThan(2000);
|
||||
void deadAt; // retained for debugging; no timing assertion (flake-prone)
|
||||
|
||||
const final = await queue.getJob(job.id);
|
||||
expect(final?.error_text).toMatch(/timeout exceeded/i);
|
||||
@@ -304,7 +310,7 @@ describeE2E('E2E: Minions resilience (OpenClaw real-world patterns)', () => {
|
||||
}, 60_000);
|
||||
|
||||
// --- 5. Cascade kill under load: cancelJob aborts all live descendants ---
|
||||
test('cascade kill: cancelJob on parent aborts 10 live children within 2s', async () => {
|
||||
test('cascade kill: cancelJob on parent aborts 10 live children', async () => {
|
||||
const { a, b } = await makeEngines();
|
||||
try {
|
||||
const queue = new MinionQueue(a);
|
||||
@@ -374,8 +380,12 @@ describeE2E('E2E: Minions resilience (OpenClaw real-world patterns)', () => {
|
||||
worker.stop();
|
||||
await startP;
|
||||
|
||||
// Correctness gate: all 10 cooperative handlers observed the abort and the
|
||||
// DB shows every descendant + root cancelled. We do NOT assert a wall-clock
|
||||
// upper bound on cancelElapsed — the 3s abort poll deadline above already
|
||||
// bounds the wait, and asserting a tighter time flakes on shared runners.
|
||||
expect(abortedChildren.size).toBe(10);
|
||||
expect(cancelElapsed).toBeLessThan(3000);
|
||||
void cancelElapsed; // retained for debugging; no timing assertion (flake-prone)
|
||||
|
||||
// DB truth: every descendant + root is 'cancelled'
|
||||
const conn = getConn();
|
||||
|
||||
@@ -78,7 +78,11 @@ describeE2E('v0.18.0 multi-source — Postgres schema shape (fresh install)', ()
|
||||
);
|
||||
expect(rows.length).toBe(1);
|
||||
expect(rows[0].is_nullable).toBe('NO');
|
||||
expect(String(rows[0].column_default)).toContain('default');
|
||||
// Postgres renders a TEXT DEFAULT 'default' literal as `'default'::text`.
|
||||
// Assert the exact stored expression rather than a loose substring so a
|
||||
// drift in the schema DEFAULT (e.g. a different sentinel source id) fails
|
||||
// here instead of silently passing.
|
||||
expect(String(rows[0].column_default)).toBe("'default'::text");
|
||||
});
|
||||
|
||||
test('composite UNIQUE pages(source_id, slug) replaces global UNIQUE(slug)', async () => {
|
||||
@@ -292,6 +296,18 @@ describeE2E('v0.18.0 multi-source — cascade delete covers every dependent row'
|
||||
`INSERT INTO files (source_id, page_id, filename, storage_path, content_hash)
|
||||
VALUES ('cascadetest', ${aliceId}, 'alice.pdf', 'cascadetest/people/alice/alice.pdf', 'fh1')`,
|
||||
);
|
||||
const aliceFile = await conn.unsafe(
|
||||
`SELECT id FROM files WHERE source_id = 'cascadetest' AND storage_path = 'cascadetest/people/alice/alice.pdf'`,
|
||||
);
|
||||
const aliceFileId = aliceFile[0].id as number;
|
||||
|
||||
// file_migration_ledger row keyed on the file (FK file_id ON DELETE
|
||||
// CASCADE). Removing the source cascades sources → files → ledger.
|
||||
await conn.unsafe(
|
||||
`INSERT INTO file_migration_ledger (file_id, storage_path_old, storage_path_new, status)
|
||||
VALUES (${aliceFileId}, 'cascadetest/people/alice/alice.pdf', 'cascadetest/people/alice/alice.pdf', 'pending')
|
||||
ON CONFLICT (file_id) DO NOTHING`,
|
||||
);
|
||||
|
||||
// Sanity: everything exists
|
||||
expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM pages WHERE source_id = 'cascadetest'`))[0].n).toBe(2);
|
||||
@@ -299,6 +315,7 @@ describeE2E('v0.18.0 multi-source — cascade delete covers every dependent row'
|
||||
expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM timeline_entries WHERE page_id = ${aliceId}`))[0].n).toBe(1);
|
||||
expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM links WHERE from_page_id = ${aliceId}`))[0].n).toBe(1);
|
||||
expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM files WHERE source_id = 'cascadetest'`))[0].n).toBe(1);
|
||||
expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM file_migration_ledger WHERE file_id = ${aliceFileId}`))[0].n).toBe(1);
|
||||
|
||||
// Remove the source.
|
||||
// v0.26.5: populated sources require --confirm-destructive; --yes alone is rejected.
|
||||
@@ -310,6 +327,7 @@ describeE2E('v0.18.0 multi-source — cascade delete covers every dependent row'
|
||||
expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM timeline_entries WHERE page_id = ${aliceId}`))[0].n).toBe(0);
|
||||
expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM links WHERE from_page_id = ${aliceId}`))[0].n).toBe(0);
|
||||
expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM files WHERE source_id = 'cascadetest'`))[0].n).toBe(0);
|
||||
expect((await conn.unsafe(`SELECT COUNT(*)::int AS n FROM file_migration_ledger WHERE file_id = ${aliceFileId}`))[0].n).toBe(0);
|
||||
|
||||
// The sources row itself is gone.
|
||||
const src = await conn.unsafe(`SELECT id FROM sources WHERE id = 'cascadetest'`);
|
||||
@@ -378,8 +396,10 @@ describeE2E('v0.18.0 multi-source — sync --source routes through sources table
|
||||
|
||||
test('performSync with no sourceId falls back to global sync.repo_path', async () => {
|
||||
const engine = getEngine();
|
||||
// Global config is still '/some/other/default/path' from the
|
||||
// previous test. Without --source, performSync uses it.
|
||||
// Self-contained: set the global config this test depends on directly
|
||||
// instead of inheriting the side effect of the previous test. Without
|
||||
// --source, performSync must read this global key.
|
||||
await engine.setConfig('sync.repo_path', '/some/other/default/path');
|
||||
let err: Error | null = null;
|
||||
try {
|
||||
await performSync(engine, {});
|
||||
|
||||
@@ -112,4 +112,27 @@ describe('v0.29 E2E — getRecentSalience (Garry test)', () => {
|
||||
const rows = await engine.getRecentSalience({ days: 7, slugPrefix: 'nope/does-not-exist/' });
|
||||
expect(rows).toEqual([]);
|
||||
});
|
||||
|
||||
// TIM-37: the daily briefing writes to the vault and re-ingests as
|
||||
// `briefings/<date>`. Without this filter the briefing itself would top
|
||||
// every subsequent Brain Pulse — self-reference with no signal.
|
||||
describe('TIM-37 — briefings excluded from their own Brain Pulse', () => {
|
||||
test('default query hides briefings/* slugs', async () => {
|
||||
await engine.putPage('briefings/2026-05-19', {
|
||||
type: 'note',
|
||||
title: 'Daily Briefing — 2026-05-19',
|
||||
compiled_truth: 'Auto-generated cron briefing.',
|
||||
});
|
||||
const rows = await engine.getRecentSalience({ days: 7, limit: 50 });
|
||||
expect(rows.some(r => r.slug.startsWith('briefings/'))).toBe(false);
|
||||
});
|
||||
|
||||
test('explicit slugPrefix=briefings/ still returns them', async () => {
|
||||
const rows = await engine.getRecentSalience({ days: 7, slugPrefix: 'briefings/' });
|
||||
expect(rows.length).toBeGreaterThan(0);
|
||||
for (const r of rows) {
|
||||
expect(r.slug.startsWith('briefings/')).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -128,6 +128,17 @@ describe('SearchResult fields', () => {
|
||||
expect(r.chunk_index).toBeDefined();
|
||||
expect(typeof r.chunk_index).toBe('number');
|
||||
});
|
||||
|
||||
test('empty keyword query returns a defined array without throwing', async () => {
|
||||
const results = await engine.searchKeyword('');
|
||||
expect(Array.isArray(results)).toBe(true);
|
||||
});
|
||||
|
||||
test('zero vector search returns a defined array without throwing', async () => {
|
||||
const zeroVector = new Float32Array(1536);
|
||||
const results = await engine.searchVector(zeroVector);
|
||||
expect(Array.isArray(results)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('detail parameter', () => {
|
||||
@@ -145,9 +156,11 @@ describe('detail parameter', () => {
|
||||
});
|
||||
|
||||
test('detail=low on vector search filters to compiled_truth', async () => {
|
||||
// Use a timeline-direction embedding — with detail=low, should get no results
|
||||
// or only compiled_truth results
|
||||
// Use a timeline-direction embedding — detail=low filters to compiled_truth.
|
||||
// Vector search returns every chunk with an embedding (ordered by distance),
|
||||
// so the seeded compiled_truth chunks are non-empty and ALL compiled_truth.
|
||||
const results = await engine.searchVector(basisEmbedding(1), { detail: 'low' });
|
||||
expect(results.length).toBeGreaterThan(0);
|
||||
for (const r of results) {
|
||||
expect(r.chunk_source).toBe('compiled_truth');
|
||||
}
|
||||
|
||||
@@ -73,7 +73,7 @@ describeE2E('E2E: Check-Update', () => {
|
||||
expect(stdout).toContain('--json');
|
||||
});
|
||||
|
||||
test('handles no-releases gracefully (current repo state)', async () => {
|
||||
test('check-update --json contract holds regardless of real release state', async () => {
|
||||
const proc = Bun.spawn(['bun', 'run', 'src/cli.ts', 'check-update', '--json'], {
|
||||
cwd: new URL('../..', import.meta.url).pathname,
|
||||
stdout: 'pipe',
|
||||
@@ -84,8 +84,16 @@ describeE2E('E2E: Check-Update', () => {
|
||||
|
||||
expect(exitCode).toBe(0);
|
||||
const output = JSON.parse(stdout);
|
||||
// With no releases, should return false and an error
|
||||
expect(output.update_available).toBe(false);
|
||||
// Don't pin update_available to a literal value — the repo may or may not
|
||||
// have a published release. Assert the JSON shape instead.
|
||||
expect(typeof output.update_available).toBe('boolean');
|
||||
expect(output.current_version).toBe(VERSION);
|
||||
if (output.latest_version != null) {
|
||||
expect(typeof output.latest_version).toBe('string');
|
||||
}
|
||||
if (output.release_url != null) {
|
||||
expect(typeof output.release_url).toBe('string');
|
||||
}
|
||||
});
|
||||
|
||||
test('version comparison wiring works end-to-end', () => {
|
||||
|
||||
@@ -134,3 +134,42 @@ describe('shared wiring helper holds the cycle lock (5A)', () => {
|
||||
expect(src).toContain('withRefreshingLock(engine, lockId');
|
||||
});
|
||||
});
|
||||
|
||||
describe('#2144: zero-yield tombstone progress semantics', () => {
|
||||
it('continues when a zero-atom batch still shrinks the backlog (tombstoned pages)', async () => {
|
||||
let batches = 0;
|
||||
const result = await runExtractAtomsDrain(
|
||||
{
|
||||
withLock: passThroughLock,
|
||||
// consumed: before#1=4, after#1=2 (<4 → progress), before#2=2,
|
||||
// after#2=0 (<2 → progress), before#3=0 → drained; final repeats 0.
|
||||
countRemaining: seq([4, 2, 2, 0, 0]),
|
||||
runBatch: async () => { batches++; return { extracted: 0, skipped: 0 }; },
|
||||
now: () => 0,
|
||||
},
|
||||
{ windowMs: 1_000_000 },
|
||||
);
|
||||
expect(result.stopped).toBe('drained');
|
||||
expect(result.batches).toBe(2);
|
||||
expect(result.extracted).toBe(0);
|
||||
expect(result.remaining).toBe(0);
|
||||
expect(batches).toBe(2);
|
||||
});
|
||||
|
||||
it('stops no_progress when a zero-atom batch leaves the backlog flat', async () => {
|
||||
let batches = 0;
|
||||
const result = await runExtractAtomsDrain(
|
||||
{
|
||||
withLock: passThroughLock,
|
||||
countRemaining: seq([5, 5]),
|
||||
runBatch: async () => { batches++; return { extracted: 0, skipped: 0 }; },
|
||||
now: () => 0,
|
||||
},
|
||||
{ windowMs: 1_000_000 },
|
||||
);
|
||||
expect(result.stopped).toBe('no_progress');
|
||||
expect(result.batches).toBe(1);
|
||||
expect(result.remaining).toBe(5);
|
||||
expect(batches).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -432,3 +432,50 @@ describe('v0.41.2.1: runPhaseExtractAtoms — dual-source merge + idempotency',
|
||||
expect(discovered.details?.atoms_extracted).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('#2144: zero-yield tombstone', () => {
|
||||
test('zero-yield page is stamped and excluded from rediscovery', async () => {
|
||||
await seedPage({ slug: 'article/zero-yield', type: 'article' });
|
||||
// Successful LLM call that yields no atoms.
|
||||
const result = await runPhaseExtractAtoms(engine, { _transcripts: [], _chat: stubChat('[]') });
|
||||
expect(result.details?.pages_processed).toBe(1);
|
||||
expect(result.details?.atoms_extracted).toBe(0);
|
||||
|
||||
// Stamp landed: atoms_scan_hash = first 16 chars of the page's content_hash.
|
||||
const rows = await engine.executeRaw<{ scan: string; ch: string }>(
|
||||
`SELECT frontmatter->>'atoms_scan_hash' AS scan, content_hash AS ch
|
||||
FROM pages WHERE slug = 'article/zero-yield'`,
|
||||
);
|
||||
expect(rows[0].scan).toBe(rows[0].ch.slice(0, 16));
|
||||
|
||||
// No longer rediscovered.
|
||||
const discovered = await discoverExtractablePages(engine, 'default');
|
||||
expect(discovered.find((d) => d.slug === 'article/zero-yield')).toBeUndefined();
|
||||
});
|
||||
|
||||
test('content change re-eligibilizes a tombstoned page', async () => {
|
||||
await seedPage({ slug: 'article/evolves', type: 'article' });
|
||||
await runPhaseExtractAtoms(engine, { _transcripts: [], _chat: stubChat('[]') });
|
||||
expect((await discoverExtractablePages(engine, 'default')).length).toBe(0);
|
||||
|
||||
// Simulate an edit: content_hash moves while the stale stamp stays.
|
||||
await engine.executeRaw(
|
||||
`UPDATE pages SET content_hash = 'fresh-hash-after-edit' WHERE slug = $1 AND source_id = 'default'`,
|
||||
['article/evolves'],
|
||||
);
|
||||
const rediscovered = await discoverExtractablePages(engine, 'default');
|
||||
expect(rediscovered.map((d) => d.slug)).toContain('article/evolves');
|
||||
});
|
||||
|
||||
test('failed chat does NOT stamp — page stays retryable', async () => {
|
||||
await seedPage({ slug: 'article/transient-failure', type: 'article' });
|
||||
const failingChat = async (_o: ChatOpts): Promise<ChatResult> => { throw new Error('rate limit'); };
|
||||
await runPhaseExtractAtoms(engine, { _transcripts: [], _chat: failingChat as never });
|
||||
const rows = await engine.executeRaw<{ scan: string | null }>(
|
||||
`SELECT frontmatter->>'atoms_scan_hash' AS scan FROM pages WHERE slug = 'article/transient-failure'`,
|
||||
);
|
||||
expect(rows[0].scan).toBeNull();
|
||||
const discovered = await discoverExtractablePages(engine, 'default');
|
||||
expect(discovered.map((d) => d.slug)).toContain('article/transient-failure');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -209,6 +209,32 @@ describe('gbrain extract --stale', () => {
|
||||
expect(usRows[0]?.eq).toBe(true);
|
||||
});
|
||||
|
||||
test('REGRESSION: page with updated_at BEFORE LINK_EXTRACTOR_VERSION_TS clears (no permanent-stale loop)', async () => {
|
||||
// The v112 watermark column ships with no backfill, so every pre-existing
|
||||
// page starts NULL-stale — and most pre-date the version bump. Pre-fix,
|
||||
// extractStaleFromDB stamped links_extracted_at = read updated_at; for a
|
||||
// page edited before LINK_EXTRACTOR_VERSION_TS the stamp landed BELOW the
|
||||
// version threshold, so the version arm (links_extracted_at < versionTs)
|
||||
// re-flagged it stale forever — an infinite re-extract loop that never
|
||||
// cleared the lag (observed: 97% of pages stuck permanently).
|
||||
await engine.putPage('people/alice', personPage('Alice'));
|
||||
await engine.putPage('companies/acme', companyPage('Acme', '[Alice](people/alice) leads [Acme](companies/acme).'));
|
||||
// Backdate every page to BEFORE the extractor version timestamp.
|
||||
await engine.executeRaw(`UPDATE pages SET updated_at = '2020-01-01T00:00:00Z'`);
|
||||
expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(2);
|
||||
|
||||
await runExtract(engine, ['--stale']);
|
||||
// Fixed: stamp = GREATEST(read updated_at, versionTs) → lifts old pages to
|
||||
// the threshold so the version arm clears, while a real future edit still
|
||||
// advances updated_at past the stamp (CDX-1 race protection preserved).
|
||||
expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(0);
|
||||
|
||||
// Second run must ALSO find 0 — the defining symptom of the bug was that it
|
||||
// never converged.
|
||||
await runExtract(engine, ['--stale']);
|
||||
expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(0);
|
||||
});
|
||||
|
||||
test('CDX-4 (D2): a link-flush throw aborts the sweep and leaves pages UNSTAMPED', async () => {
|
||||
await engine.putPage('people/alice', personPage('Alice'));
|
||||
await engine.putPage('companies/acme', companyPage('Acme', '[Alice](people/alice) founded [Acme](companies/acme).'));
|
||||
|
||||
@@ -140,6 +140,17 @@ describe('extractEntityRefs', () => {
|
||||
expect(wikiRefs[0].needsResolution).toBe(true);
|
||||
});
|
||||
|
||||
test('recognizes reference-page wikilinks as concrete targets', () => {
|
||||
const refs = extractEntityRefs('See [[reference/mcminnville-market-data]] for source context.');
|
||||
expect(refs.length).toBe(1);
|
||||
expect(refs[0]).toMatchObject({
|
||||
name: 'reference/mcminnville-market-data',
|
||||
slug: 'reference/mcminnville-market-data',
|
||||
dir: 'reference',
|
||||
});
|
||||
expect(refs[0].needsResolution).toBeUndefined();
|
||||
});
|
||||
|
||||
test('skips qualified-syntax tokens (those belong to 2a)', () => {
|
||||
// [[wiki:topics/ai]] looks like 2a's qualified shape — even though
|
||||
// it wouldn't satisfy DIR_PATTERN, 2c must not claim it either
|
||||
@@ -1236,6 +1247,43 @@ describe('makeResolver — fallback chain', () => {
|
||||
const out = await r.resolveBasenameMatches!('struktura');
|
||||
expect(out.sort()).toEqual(['notes/struktura', 'struktura']);
|
||||
});
|
||||
|
||||
test('opts.sourceId is forwarded to findByTitleFuzzy (twin of #1436 fix)', async () => {
|
||||
// Captures every (name, dirPrefix, minSimilarity, sourceId) call so we
|
||||
// can assert the resolver threads sourceId through. Without the wire-up,
|
||||
// findByTitleFuzzy would be called with sourceId=undefined and the SQL
|
||||
// could return cross-source slug suggestions that the FK filter
|
||||
// downstream silently drops.
|
||||
const calls: Array<{ name: string; dirPrefix?: string; minSimilarity?: number; sourceId?: string }> = [];
|
||||
const engine = {
|
||||
async getPage() { return null; },
|
||||
async findByTitleFuzzy(name: string, dirPrefix?: string, minSimilarity?: number, sourceId?: string) {
|
||||
calls.push({ name, dirPrefix, minSimilarity, sourceId });
|
||||
return null;
|
||||
},
|
||||
async searchKeyword() { return []; },
|
||||
} as unknown as BrainEngine;
|
||||
const r = makeResolver(engine, { mode: 'batch', sourceId: 'src-a' });
|
||||
await r.resolve('Alice Example', 'people');
|
||||
expect(calls.length).toBeGreaterThan(0);
|
||||
expect(calls.every(c => c.sourceId === 'src-a')).toBe(true);
|
||||
});
|
||||
|
||||
test('opts.sourceId omitted → findByTitleFuzzy receives undefined (back-compat)', async () => {
|
||||
const calls: Array<{ sourceId?: string }> = [];
|
||||
const engine = {
|
||||
async getPage() { return null; },
|
||||
async findByTitleFuzzy(_name: string, _dirPrefix?: string, _min?: number, sourceId?: string) {
|
||||
calls.push({ sourceId });
|
||||
return null;
|
||||
},
|
||||
async searchKeyword() { return []; },
|
||||
} as unknown as BrainEngine;
|
||||
const r = makeResolver(engine, { mode: 'batch' });
|
||||
await r.resolve('Alice Example', 'people');
|
||||
expect(calls.length).toBeGreaterThan(0);
|
||||
expect(calls.every(c => c.sourceId === undefined)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('FRONTMATTER_LINK_MAP integrity', () => {
|
||||
|
||||
@@ -32,6 +32,29 @@ describe('lintContent', () => {
|
||||
expect(issues.some(i => i.rule === 'code-fence-wrap')).toBe(true);
|
||||
});
|
||||
|
||||
test('no false positive: page CONTAINS an inner ```markdown code block', () => {
|
||||
// Real-world case: a docs/SKILL page that shows a markdown example inline.
|
||||
// Before this fix, the detector used the /m flag so ^/$ matched start/end
|
||||
// of any line, which fired on any file that simply contained a ```markdown
|
||||
// line. But fixContent's regex has no /m flag and can only strip whole-file
|
||||
// wrappers, so the issue was reported as "fixable: true" yet never fixed.
|
||||
const content =
|
||||
'---\ntitle: Skill\n---\n\n# Skill\n\nExample input shape:\n\n' +
|
||||
'```markdown\n# Inner page\nContent.\n```\n\nThat ends the example.\n';
|
||||
const issues = lintContent(content, 'test.md');
|
||||
expect(issues.filter(i => i.rule === 'code-fence-wrap')).toHaveLength(0);
|
||||
});
|
||||
|
||||
test('no false positive: multiple inner ```markdown blocks', () => {
|
||||
// Documentation pages frequently include several markdown examples.
|
||||
const content =
|
||||
'---\ntitle: Examples\n---\n\n# Examples\n\nFirst:\n\n' +
|
||||
'```markdown\nfoo\n```\n\nSecond:\n\n' +
|
||||
'```markdown\nbar\n```\n\nDone.\n';
|
||||
const issues = lintContent(content, 'test.md');
|
||||
expect(issues.filter(i => i.rule === 'code-fence-wrap')).toHaveLength(0);
|
||||
});
|
||||
|
||||
test('detects placeholder dates', () => {
|
||||
const content = '---\ntitle: Test\ntype: person\ncreated: YYYY-MM-DD\n---\n\n# Test';
|
||||
const issues = lintContent(content, 'test.md');
|
||||
|
||||
@@ -9,6 +9,7 @@ import { loadConfigWithEngine, type GBrainConfig } from '../src/core/config.ts';
|
||||
|
||||
interface FakeEngine {
|
||||
getConfig(key: string): Promise<string | null | undefined>;
|
||||
listConfigKeys?(prefix: string): Promise<string[]>;
|
||||
}
|
||||
|
||||
function makeEngine(map: Record<string, string | null | undefined>): FakeEngine {
|
||||
@@ -16,6 +17,9 @@ function makeEngine(map: Record<string, string | null | undefined>): FakeEngine
|
||||
async getConfig(key: string) {
|
||||
return map[key];
|
||||
},
|
||||
async listConfigKeys(prefix: string) {
|
||||
return Object.keys(map).filter(key => key.startsWith(prefix));
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -92,6 +96,31 @@ describe('loadConfigWithEngine (Phase 4 / F3)', () => {
|
||||
expect(merged?.embedding_image_ocr).toBe(true);
|
||||
});
|
||||
|
||||
test('DB provider_base_urls.<provider> fills the gateway base URL map', async () => {
|
||||
const base: GBrainConfig = { engine: 'pglite' };
|
||||
const engine = makeEngine({
|
||||
'provider_base_urls.llama-server-reranker': 'http://127.0.0.1:8091/v1',
|
||||
});
|
||||
const merged = await loadConfigWithEngine(engine, base);
|
||||
expect(merged?.provider_base_urls?.['llama-server-reranker']).toBe('http://127.0.0.1:8091/v1');
|
||||
});
|
||||
|
||||
test('provider_base_urls merge is per-provider: file value wins and DB fills siblings', async () => {
|
||||
const base: GBrainConfig = {
|
||||
engine: 'pglite',
|
||||
provider_base_urls: {
|
||||
'llama-server-reranker': 'http://file.example/v1',
|
||||
},
|
||||
};
|
||||
const engine = makeEngine({
|
||||
'provider_base_urls.llama-server-reranker': 'http://db.example/v1',
|
||||
'provider_base_urls.openrouter': 'http://openrouter.example/v1',
|
||||
});
|
||||
const merged = await loadConfigWithEngine(engine, base);
|
||||
expect(merged?.provider_base_urls?.['llama-server-reranker']).toBe('http://file.example/v1');
|
||||
expect(merged?.provider_base_urls?.openrouter).toBe('http://openrouter.example/v1');
|
||||
});
|
||||
|
||||
test('engine.getConfig throwing is non-fatal — file/env config still returned', async () => {
|
||||
const base: GBrainConfig = {
|
||||
engine: 'pglite',
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { getRecipe } from '../src/core/ai/recipes/index.ts';
|
||||
|
||||
describe('OpenRouter recipe — reranker touchpoint', () => {
|
||||
test('declares a reranker touchpoint', () => {
|
||||
const r = getRecipe('openrouter');
|
||||
expect(r).toBeDefined();
|
||||
expect(r!.touchpoints.reranker).toBeDefined();
|
||||
});
|
||||
|
||||
test('models list includes all supported IDs (incl. NVIDIA :free suffix)', () => {
|
||||
const m = getRecipe('openrouter')!.touchpoints.reranker!.models;
|
||||
expect(m).toContain('cohere/rerank-v3.5');
|
||||
expect(m).toContain('cohere/rerank-4-fast');
|
||||
expect(m).toContain('cohere/rerank-4-pro');
|
||||
// The :free suffix must appear in full — gateway.rerank() does exact
|
||||
// string matching against the allowlist (no v0.31.12 extended-model bypass
|
||||
// on the rerank path), so truncating to `nvidia/.../v2` would 403.
|
||||
expect(m).toContain('nvidia/llama-nemotron-rerank-vl-1b-v2:free');
|
||||
});
|
||||
|
||||
test('default_model is cohere/rerank-v3.5', () => {
|
||||
const tp = getRecipe('openrouter')!.touchpoints.reranker!;
|
||||
expect(tp.default_model).toBe('cohere/rerank-v3.5');
|
||||
expect(tp.models).toContain(tp.default_model);
|
||||
});
|
||||
|
||||
test('path is /rerank (NOT ZeroEntropy default /models/rerank)', () => {
|
||||
const tp = getRecipe('openrouter')!.touchpoints.reranker!;
|
||||
expect(tp.path).toBe('/rerank');
|
||||
});
|
||||
|
||||
test('max_payload_bytes and timeout match plan', () => {
|
||||
const tp = getRecipe('openrouter')!.touchpoints.reranker!;
|
||||
expect(tp.max_payload_bytes).toBe(5_000_000);
|
||||
expect(tp.default_timeout_ms).toBe(5_000);
|
||||
});
|
||||
|
||||
test('cost_per_1m_tokens_usd is set (pseudo-rate for per-search billing)', () => {
|
||||
const tp = getRecipe('openrouter')!.touchpoints.reranker!;
|
||||
expect(typeof tp.cost_per_1m_tokens_usd).toBe('number');
|
||||
expect(tp.cost_per_1m_tokens_usd).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
@@ -218,15 +218,21 @@ describe('progress reporter', () => {
|
||||
test('only one process-level signal handler installed across many reporters', () => {
|
||||
// Baseline: one handler already installed by prior tests in this file.
|
||||
const installedBefore = __signalHandlerInstalledForTest();
|
||||
// liveReporters is module-global, so a reporter left running by ANOTHER
|
||||
// test file in the same shard shows up here. Assert the DELTA (these 50
|
||||
// lifecycles leak nothing) instead of an absolute zero — the absolute
|
||||
// form flaked whenever shard composition changed and an unrelated file
|
||||
// held a live reporter across this test.
|
||||
const liveBefore = __liveReporterCountForTest();
|
||||
const { stream } = sink(false);
|
||||
for (let i = 0; i < 50; i++) {
|
||||
const p = createProgress({ mode: 'json', stream, minIntervalMs: 0, minItems: 1 });
|
||||
p.start(`phase_${i}`, 1);
|
||||
p.finish();
|
||||
}
|
||||
// After 50 reporter lifecycles, still exactly one handler and zero leaked live entries.
|
||||
// After 50 reporter lifecycles, still exactly one handler and no new live entries.
|
||||
expect(__signalHandlerInstalledForTest()).toBe(installedBefore || true);
|
||||
expect(__liveReporterCountForTest()).toBe(0);
|
||||
expect(__liveReporterCountForTest()).toBe(liveBefore);
|
||||
});
|
||||
|
||||
test('startHeartbeat() fires heartbeats and stop() clears', async () => {
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { stripGapsSection } from '../src/core/think/index.ts';
|
||||
import { buildThinkSystemPrompt } from '../src/core/think/prompt.ts';
|
||||
|
||||
// `gbrain think` returns gaps in the structured `gaps` array, which both the
|
||||
// CLI (`src/commands/think.ts`) and the persisted synthesis page
|
||||
// (`persistSynthesis`) render exactly once. Older prompts also asked for a
|
||||
// "Gaps" section inside the answer prose, so a model that still emits one made
|
||||
// the output print "## Gaps" twice. `stripGapsSection` removes the prose
|
||||
// section so the structured array is the single source of truth.
|
||||
|
||||
describe('stripGapsSection', () => {
|
||||
test('removes a trailing "## Gaps" section', () => {
|
||||
const answer = 'The answer with a claim [people/alice].\n\n## Gaps\n- no update since 2026-03-22 [projects/acme]\n- pricing not recorded';
|
||||
const out = stripGapsSection(answer);
|
||||
expect(out).not.toContain('## Gaps');
|
||||
expect(out).not.toContain('no update since');
|
||||
expect(out).toContain('The answer with a claim [people/alice].');
|
||||
});
|
||||
|
||||
test('removes a level-3 "### Gaps" section', () => {
|
||||
const out = stripGapsSection('Body text.\n\n### Gaps\n- missing thing');
|
||||
expect(out).not.toMatch(/#+\s+Gaps/i);
|
||||
expect(out).toBe('Body text.');
|
||||
});
|
||||
|
||||
test('is case-insensitive', () => {
|
||||
expect(stripGapsSection('Body.\n\n## GAPS\n- x')).toBe('Body.');
|
||||
expect(stripGapsSection('Body.\n\n## gaps\n- x')).toBe('Body.');
|
||||
});
|
||||
|
||||
test('returns the answer unchanged when there is no Gaps section', () => {
|
||||
const answer = 'Just an answer.\n\n## Conflicts\n- a vs b';
|
||||
expect(stripGapsSection(answer)).toBe(answer);
|
||||
});
|
||||
|
||||
test('does not match a heading that merely starts with "Gaps"', () => {
|
||||
const answer = 'Body.\n\n## Gaps in the coverage\n- this is real content';
|
||||
expect(stripGapsSection(answer)).toBe(answer);
|
||||
});
|
||||
|
||||
test('stops at the next same-or-higher heading (preserves later content)', () => {
|
||||
const answer = 'Intro.\n\n## Gaps\n- missing x\n\n## Sources\n- [a]';
|
||||
const out = stripGapsSection(answer);
|
||||
expect(out).not.toContain('missing x');
|
||||
expect(out).toContain('## Sources');
|
||||
expect(out).toContain('- [a]');
|
||||
});
|
||||
|
||||
test('handles empty / falsy input', () => {
|
||||
expect(stripGapsSection('')).toBe('');
|
||||
});
|
||||
|
||||
test('the bug repro: strip + structured render yields exactly one "## Gaps"', () => {
|
||||
// Mirrors the render in src/commands/think.ts: print the (stripped) answer,
|
||||
// then append one "## Gaps" block from the structured `gaps` array.
|
||||
const answer = 'Answer prose [people/alice].\n\n## Gaps\n- the prose gap, slightly different wording';
|
||||
const gaps = ['the structured gap'];
|
||||
const rendered =
|
||||
stripGapsSection(answer) + '\n\n## Gaps\n' + gaps.map((g) => `- ${g}`).join('\n');
|
||||
expect((rendered.match(/## Gaps/g) ?? []).length).toBe(1);
|
||||
expect(rendered).toContain('the structured gap');
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildThinkSystemPrompt — gaps go in the structured array, not the answer body', () => {
|
||||
test('the answer schema no longer lists "Gaps" as a body section', () => {
|
||||
const out = buildThinkSystemPrompt({});
|
||||
expect(out).not.toContain('Sections: Answer, Conflicts (optional), Gaps');
|
||||
expect(out).toContain('gaps belong in the gaps array');
|
||||
});
|
||||
|
||||
test('still requires the structured "gaps" array', () => {
|
||||
const out = buildThinkSystemPrompt({});
|
||||
expect(out).toContain('"gaps"');
|
||||
});
|
||||
|
||||
test('preserves the Conflicts section and the Hard rules', () => {
|
||||
const out = buildThinkSystemPrompt({});
|
||||
expect(out).toContain('Conflicts');
|
||||
expect(out).toContain('Hard rules:');
|
||||
expect(out).toContain('Cite EVERY substantive claim');
|
||||
});
|
||||
|
||||
test('willSave mode routes gaps to the structured array (no body Gaps section)', () => {
|
||||
const out = buildThinkSystemPrompt({ willSave: true });
|
||||
expect(out).not.toContain('cover Answer, Conflicts, and Gaps thoroughly');
|
||||
expect(out).toContain('structured "gaps" array');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user