From ab4e31e30a24ecaaddce12e18b64f691b7dbec76 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Fri, 12 Jun 2026 12:06:22 -0700 Subject: [PATCH] feat(eval): gbrain eval brainbench CLI + run-all wiring (EvalRunRecord v3) No-DB cli.ts route (brings its own PGLite), --out as the canonical CI artifact, explicit grace-tick process.exit(verdict) (PGLite exitCode-hijack + Bun stdout-discard guards), anti-vacuous-pass + seed-failure exit-2 paths, _meta.metric_glossary block, --update-baseline canonical writer, --compare with main-baseline governance. eval run-all's brainbench stub is now wired in-process: one record per sweep, schema_version 3, mode 'n/a'. Subprocess e2e pins literal exit codes 0/1/2 end to end. --- src/cli.ts | 17 ++ src/commands/eval-brainbench.ts | 428 +++++++++++++++++++++++++++++++ src/commands/eval-run-all.ts | 47 +++- src/commands/eval.ts | 8 + test/eval-brainbench-e2e.test.ts | 214 ++++++++++++++++ test/eval-run-all.test.ts | 26 +- 6 files changed, 734 insertions(+), 6 deletions(-) create mode 100644 src/commands/eval-brainbench.ts create mode 100644 test/eval-brainbench-e2e.test.ts diff --git a/src/cli.ts b/src/cli.ts index 09d187cea..1fef86fc2 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -1288,6 +1288,23 @@ async function handleCliOnly(command: string, args: string[]) { process.exit(await runReplayNoBrain(args.slice(2))); } + // BrainBench brings its own in-memory PGLite (longmemeval pattern) and is + // hermetic by default — no gateway, no user brain, no config required. The + // command owns its exit codes (0 pass / 1 regression / 2 error) and exits + // explicitly via its grace-tick exit path (PGLite exitCode-hijack guard). + if (command === 'eval' && args[0] === 'brainbench') { + const { runEvalBrainBench } = await import('./commands/eval-brainbench.ts'); + if (args.includes('--llm') && !args.includes('--help') && !args.includes('-h')) { + // --llm is the one mode that talks to a provider; mirror the + // longmemeval gateway bootstrap so extraction calls are priced. + const config = loadConfig() ?? ({} as GBrainConfig); + const { configureGateway } = await import('./core/ai/gateway.ts'); + configureGateway(buildGatewayConfig(config)); + } + await runEvalBrainBench(args.slice(1)); + return; // unreachable — runEvalBrainBench always exits — but keeps control flow explicit + } + // v0.28.8: longmemeval brings its own in-memory PGLite. Bypassing // connectEngine here keeps `gbrain eval longmemeval --help` and benchmark // runs working on machines that have no `~/.gbrain/config.json` configured. diff --git a/src/commands/eval-brainbench.ts b/src/commands/eval-brainbench.ts new file mode 100644 index 000000000..228163844 --- /dev/null +++ b/src/commands/eval-brainbench.ts @@ -0,0 +1,428 @@ +/** + * `gbrain eval brainbench` — the cross-harness memory conformance suite + * (Cathedral 2; docs/eval/BRAINBENCH.md). + * + * Brings its own in-memory PGLite (longmemeval pattern) — the cli.ts + * dispatcher routes here BEFORE connectEngine, so no user brain is ever + * touched and `--help` works with no DB. + * + * Exit contract (decision 9 — the exit code IS the CI product): + * 0 pass · 1 regression · 2 error / inconclusive / usage + * PGLite (Emscripten) writes its WASM status into process.exitCode at + * arbitrary event-loop ticks, and Bun discards queued stdout on + * process.exit — so the verdict lives in a local variable, `--out FILE` is + * the canonical CI artifact (written sync before exit), and exit happens + * explicitly after a short grace tick that lets stdout drain. + */ + +import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { execSync } from 'node:child_process'; +import { cliOptsToProgressOptions, getCliOptions } from '../core/cli-options.ts'; +import { createProgress } from '../core/progress.ts'; +import { buildMetricGlossaryMeta } from '../core/eval/metric-glossary.ts'; +import { isAvailable } from '../core/ai/gateway.ts'; +import { FixtureValidationError, loadCorpus } from '../eval/brainbench/fixtures.ts'; +import { runBrainBench } from '../eval/brainbench/harness.ts'; +import { + compareBaselines, + parseBaseline, + renderScoreboardMarkdown, + serializeBaseline, + toCanonicalBaseline, +} from '../eval/brainbench/scoreboard.ts'; +import { + ALL_HARNESSES, + ALL_SUITES, + RESULT_SCHEMA_VERSION, + type BrainBenchBaseline, + type BrainBenchResult, + type BrainBenchSuite, + type CompareOutcome, + type HarnessName, +} from '../eval/brainbench/types.ts'; + +export const DEFAULT_FIXTURES_DIR = 'evals/brainbench/fixtures'; +export const DEFAULT_GOLD_DIR = 'evals/brainbench/gold'; +export const DEFAULT_BASELINE_PATH = 'evals/brainbench/baselines/main.json'; +const DEFAULT_LLM_BUDGET_USD = 5; + +function usage(): void { + process.stderr.write( + `Usage: gbrain eval brainbench [options]\n\n` + + `Cross-harness memory conformance suite. Hermetic by default: in-memory\n` + + `PGLite, no API keys, no LLM calls. See docs/eval/BRAINBENCH.md.\n\n` + + `Options:\n` + + ` --fixtures DIR Fixture corpus (default: ${DEFAULT_FIXTURES_DIR}).\n` + + ` --gold DIR Sealed gold dir (default: ${DEFAULT_GOLD_DIR}).\n` + + ` --harness a,b | all Harness seams to grade (default: all).\n` + + ` --suite a,b | all Suites to run (default: all).\n` + + ` --include-holdout Score holdout fixtures too (published-run mode).\n` + + ` --json JSON result on stdout instead of the markdown scoreboard.\n` + + ` --out FILE Write the full JSON result to FILE (canonical CI artifact).\n` + + ` --compare BASE [CURRENT] Gate against BASE baseline. With CURRENT: pure\n` + + ` file-vs-file diff, no run. CI passes MAIN's baseline\n` + + ` as BASE (git show origin/master:${DEFAULT_BASELINE_PATH}).\n` + + ` --committed-baseline FILE Bless-mode verification target (default: ${DEFAULT_BASELINE_PATH}).\n` + + ` --update-baseline [FILE] Write this run as the canonical committed baseline.\n` + + ` --justification "reason" Recorded in the baseline written by --update-baseline\n` + + ` (REQUIRED by the gate when blessing a regression).\n` + + ` --allow-regression "reason" One-off escape hatch; reason recorded in the output.\n` + + ` --llm Write-back suite uses the real LLM extractor.\n` + + ` --budget-usd N Spend cap for --llm (default: $${DEFAULT_LLM_BUDGET_USD}).\n` + + ` --seed N Recorded in the receipt (default: 42).\n` + + ` -h, --help Show this help.\n\n` + + `Exit codes: 0 pass · 1 regression · 2 error/inconclusive/usage.\n`, + ); +} + +interface Args { + fixtures: string; + gold: string; + harnesses: HarnessName[]; + suites: BrainBenchSuite[]; + includeHoldout: boolean; + json: boolean; + out: string | null; + compare: string[] | null; + committedBaseline: string; + updateBaseline: string | null; + justification: string | null; + allowRegression: string | null; + llm: boolean; + budgetUsd: number; + seed: number; +} + +function parseArgs(argv: string[]): Args | { usageError: string } { + const args: Args = { + fixtures: DEFAULT_FIXTURES_DIR, + gold: DEFAULT_GOLD_DIR, + harnesses: [...ALL_HARNESSES], + suites: [...ALL_SUITES], + includeHoldout: false, + json: false, + out: null, + compare: null, + committedBaseline: DEFAULT_BASELINE_PATH, + updateBaseline: null, + justification: null, + allowRegression: null, + llm: false, + budgetUsd: DEFAULT_LLM_BUDGET_USD, + seed: 42, + }; + const need = (flag: string, v: string | undefined): string => { + if (v === undefined || v.startsWith('--')) throw new Error(`${flag} requires a value`); + return v; + }; + try { + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + switch (a) { + case '-h': + case '--help': + return { usageError: '' }; + case '--fixtures': + args.fixtures = need(a, argv[++i]); + break; + case '--gold': + args.gold = need(a, argv[++i]); + break; + case '--harness': { + const v = need(a, argv[++i]); + if (v !== 'all') { + const list = v.split(',').map((s) => s.trim()).filter(Boolean); + for (const h of list) { + if (!(ALL_HARNESSES as readonly string[]).includes(h)) { + return { usageError: `unknown harness "${h}" (valid: ${ALL_HARNESSES.join(', ')}, all)` }; + } + } + args.harnesses = list as HarnessName[]; + } + break; + } + case '--suite': { + const v = need(a, argv[++i]); + if (v !== 'all') { + const list = v.split(',').map((s) => s.trim()).filter(Boolean); + for (const s of list) { + if (!(ALL_SUITES as readonly string[]).includes(s)) { + return { usageError: `unknown suite "${s}" (valid: ${ALL_SUITES.join(', ')}, all)` }; + } + } + args.suites = list as BrainBenchSuite[]; + } + break; + } + case '--include-holdout': + args.includeHoldout = true; + break; + case '--json': + args.json = true; + break; + case '--out': + args.out = need(a, argv[++i]); + break; + case '--compare': { + const files = [need(a, argv[++i])]; + if (argv[i + 1] && !argv[i + 1].startsWith('--')) files.push(argv[++i]); + args.compare = files; + break; + } + case '--committed-baseline': + args.committedBaseline = need(a, argv[++i]); + break; + case '--update-baseline': + args.updateBaseline = + argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[++i] : DEFAULT_BASELINE_PATH; + break; + case '--justification': + args.justification = need(a, argv[++i]); + break; + case '--allow-regression': + args.allowRegression = need(a, argv[++i]); + break; + case '--llm': + args.llm = true; + break; + case '--budget-usd': + args.budgetUsd = Number(need(a, argv[++i])); + if (!Number.isFinite(args.budgetUsd) || args.budgetUsd <= 0) { + return { usageError: '--budget-usd must be a positive number' }; + } + break; + case '--seed': + args.seed = Number(need(a, argv[++i])); + if (!Number.isInteger(args.seed)) return { usageError: '--seed must be an integer' }; + break; + default: + return { usageError: `unknown flag ${a}` }; + } + } + } catch (err) { + return { usageError: (err as Error).message }; + } + return args; +} + +function gitHeadSha(): string { + try { + return execSync('git rev-parse HEAD', { encoding: 'utf-8' }).trim(); + } catch { + return 'unknown'; + } +} + +/** + * Bun discards queued stdout on process.exit and PGLite stomps + * process.exitCode on its own ticks — hold the verdict locally, give the + * event loop one grace window to drain pipes, then exit explicitly + * (learnings pglite-emscripten-hijacks-process-exitcode + + * bun-pipe-stdout-only-flushes-while-alive). + */ +async function exitWith(code: 0 | 1 | 2): Promise { + await new Promise((resolve) => setTimeout(resolve, 300)); + process.exit(code); +} + +function readBaselineFile(path: string): BrainBenchBaseline { + return parseBaseline(readFileSync(path, 'utf-8'), path); +} + +function verdictExit(outcome: CompareOutcome): 0 | 1 | 2 { + switch (outcome.verdict) { + case 'pass': + return 0; + case 'regression': + return 1; + case 'inconclusive': + return 2; + } +} + +export async function runEvalBrainBench(argv: string[]): Promise { + const parsed = parseArgs(argv); + if ('usageError' in parsed) { + if (parsed.usageError) process.stderr.write(`eval brainbench: ${parsed.usageError}\n\n`); + usage(); + return exitWith(2); + } + const args = parsed; + + // Pure file-vs-file compare: no run, no DB. + if (args.compare && args.compare.length === 2) { + let outcome: CompareOutcome; + try { + const base = readBaselineFile(args.compare[0]); + const current = readBaselineFile(args.compare[1]); + outcome = compareBaselines(current, base, { + allowRegression: args.allowRegression ?? undefined, + }); + } catch (err) { + process.stderr.write(`eval brainbench: ${(err as Error).message}\n`); + return exitWith(2); + } + process.stdout.write(JSON.stringify(outcome, null, 2) + '\n'); + return exitWith(verdictExit(outcome)); + } + + // --llm needs a configured chat touchpoint BEFORE we spend a run on it. + if (args.llm && !isAvailable('chat')) { + process.stderr.write( + 'eval brainbench: --llm requires a configured chat model (set ANTHROPIC_API_KEY or run `gbrain models`). ' + + 'Drop --llm for the deterministic hermetic mode.\n', + ); + return exitWith(2); + } + + const reporter = createProgress(cliOptsToProgressOptions(getCliOptions())); + let result: BrainBenchResult; + try { + const corpus = await loadCorpus(args.fixtures, args.gold); + if (corpus.fixtures.length === 0) { + process.stderr.write(`eval brainbench: no fixtures found in ${args.fixtures} — refusing a vacuous pass.\n`); + return exitWith(2); + } + // No total: continuity pairs replay per (writer,reader) ordering, so the + // tick count exceeds the fixture count — a percentage would lie. + reporter.start('eval.brainbench'); + const run = await runBrainBench(corpus, { + harnesses: args.harnesses, + suites: args.suites, + includeHoldout: args.includeHoldout, + llm: args.llm, + budgetUsd: args.budgetUsd, + onProgress: (note) => reporter.tick(1, note), + }); + reporter.finish(); + + // Seed failures FIRST: an all-fixtures-failed-to-seed run would otherwise + // be misdiagnosed as "filters matched nothing" (same exit, wrong story). + if (run.seed_failures.length > 0) { + process.stderr.write(`eval brainbench: SEED FAILURES (${run.seed_failures.length}) — run invalid:\n`); + for (const f of run.seed_failures) { + process.stderr.write(` - ${f.fixture_id}: ${f.error}\n`); + } + } else if (run.fixtures_run === 0 || run.cells.length === 0) { + process.stderr.write( + 'eval brainbench: the harness/suite filters matched zero fixtures — refusing a vacuous pass.\n', + ); + return exitWith(2); + } + + const metricNames = [...new Set(run.cells.flatMap((c) => Object.keys(c.metrics)))].sort(); + result = { + receipt: { + result_schema_version: RESULT_SCHEMA_VERSION, + fixtures_hash: corpus.fixtures_hash, + harness_sha: gitHeadSha(), + ts: new Date().toISOString(), + cmd_args: argv, + seed: args.seed, + include_holdout: args.includeHoldout, + llm: args.llm, + }, + cells: run.cells, + turn_rows: run.turn_rows, + seed_failures: run.seed_failures, + _meta: { metric_glossary: buildMetricGlossaryMeta(metricNames) }, + }; + } catch (err) { + reporter.finish('aborted'); + const prefix = err instanceof FixtureValidationError ? 'fixture validation' : 'run failed'; + process.stderr.write(`eval brainbench: ${prefix}: ${(err as Error).message}\n`); + return exitWith(2); + } + + // Decide verdict BEFORE emitting (seed failures invalidate everything). + let exitCode: 0 | 1 | 2 = 0; + let compareOutcome: CompareOutcome | null = null; + + if (result.seed_failures.length > 0) { + exitCode = 2; + } else if (args.compare) { + try { + const main = readBaselineFile(args.compare[0]); + const current = toCanonicalBaseline(result); + let committed: BrainBenchBaseline | null = null; + if (current.fixtures_hash !== main.fixtures_hash && existsSync(args.committedBaseline)) { + committed = readBaselineFile(args.committedBaseline); + } + compareOutcome = compareBaselines(current, main, { + allowRegression: args.allowRegression ?? undefined, + committedBaseline: committed, + }); + exitCode = verdictExit(compareOutcome); + } catch (err) { + process.stderr.write(`eval brainbench: compare failed: ${(err as Error).message}\n`); + exitCode = 2; + } + } + + if (args.updateBaseline) { + const baseline = toCanonicalBaseline(result, args.justification ?? undefined); + mkdirSync(dirname(args.updateBaseline), { recursive: true }); + writeFileSync(args.updateBaseline, serializeBaseline(baseline)); + process.stderr.write(`[eval brainbench] baseline written: ${args.updateBaseline}\n`); + } + + // Canonical CI artifact first (sync write completes before any exit). + if (args.out) { + const outDoc = { ...result, compare: compareOutcome ?? undefined }; + mkdirSync(dirname(args.out), { recursive: true }); + writeFileSync(args.out, JSON.stringify(outDoc, null, 2) + '\n'); + process.stderr.write(`[eval brainbench] result written: ${args.out}\n`); + } + + if (args.json) { + process.stdout.write(JSON.stringify({ ...result, compare: compareOutcome ?? undefined }) + '\n'); + } else { + process.stdout.write(renderScoreboardMarkdown(result, compareOutcome)); + } + + return exitWith(exitCode); +} + +/** + * In-process entry for `gbrain eval run-all` (decision 16): full hermetic run + * over the default corpus, NO process.exit, returns the per-cell metrics for + * the EvalRunRecord. BrainBench is search-mode-independent, so run-all calls + * this once per sweep and records it under mode 'n/a'. + */ +export async function runBrainBenchCore(): Promise<{ + status: 'completed' | 'failed'; + fixtures_hash?: string; + cells?: Record>; + error?: string; +}> { + try { + const corpus = await loadCorpus(DEFAULT_FIXTURES_DIR, DEFAULT_GOLD_DIR); + if (corpus.fixtures.length === 0) { + return { status: 'failed', error: `no fixtures in ${DEFAULT_FIXTURES_DIR}` }; + } + const run = await runBrainBench(corpus, { + harnesses: [...ALL_HARNESSES], + suites: [...ALL_SUITES], + includeHoldout: false, + llm: false, + }); + if (run.seed_failures.length > 0) { + return { + status: 'failed', + fixtures_hash: corpus.fixtures_hash, + error: `seed failures: ${run.seed_failures.map((f) => f.fixture_id).join(', ')}`, + }; + } + const cells: Record> = {}; + for (const c of run.cells) { + cells[`${c.harness}/${c.suite}`] = { ...c.metrics, gold_failed: c.gold_failed, gold_total: c.gold_total }; + } + return { status: 'completed', fixtures_hash: corpus.fixtures_hash, cells }; + } catch (err) { + return { status: 'failed', error: (err as Error).message }; + } +} + +/** Test hook: everything above process.exit, without exiting. */ +export const _internal = { parseArgs, DEFAULT_BASELINE_PATH: join(DEFAULT_BASELINE_PATH) }; diff --git a/src/commands/eval-run-all.ts b/src/commands/eval-run-all.ts index b5c952077..ede0e0207 100644 --- a/src/commands/eval-run-all.ts +++ b/src/commands/eval-run-all.ts @@ -131,11 +131,18 @@ function printHelp(): void { } export interface EvalRunRecord { - schema_version: 2; + /** + * v3 (BrainBench wave, decision 16): `mode` widened to `SearchMode | 'n/a'` + * for search-mode-independent suites — brainbench runs once per sweep and + * records 'n/a' instead of fabricating a mode. v2 records (mode always a + * SearchMode) parse fine under v3 readers; eval-compare renders 'n/a' rows + * un-grouped. + */ + schema_version: 3; run_id: string; ran_at: string; suite: ValidSuite; - mode: SearchMode; + mode: SearchMode | 'n/a'; commit: string; seed: number; limit?: number; @@ -314,11 +321,45 @@ export async function runEvalRunAll(_engine: BrainEngine | null, args: string[]) // manually with the documented --mode flags and uses persistRunRecord // to log each completion. The methodology doc names this explicitly. for (const suite of opts.suites) { + // BrainBench is search-mode-independent (decision 16): run ONCE per + // sweep, in-process, recorded under mode 'n/a' — never multiplied by + // modes. The other suites keep the documented operator-runs-CLI stub. + if (suite === 'brainbench') { + const startedAt = Date.now(); + const runId = `${commit}-brainbench-na-${opts.seed}`; + const { runBrainBenchCore } = await import('./eval-brainbench.ts'); + const core = await runBrainBenchCore(); + const record: EvalRunRecord = { + schema_version: 3, + run_id: runId, + ran_at: new Date().toISOString(), + suite: 'brainbench', + mode: 'n/a', + commit, + seed: opts.seed, + limit: opts.limit, + params: { + budget_usd_retrieval: opts.budgetUsdRetrieval, + budget_usd_answer: opts.budgetUsdAnswer, + parallel: opts.parallel, + fixtures_hash: core.fixtures_hash, + cells: core.cells, + }, + status: core.status, + duration_ms: Date.now() - startedAt, + }; + if (core.error) record.error = core.error; + persistRunRecord(repoRoot, record, opts.outputDir); + if (!opts.jsonOutput) { + process.stderr.write(`[eval run-all] ${runId}: ${record.status}\n`); + } + continue; + } for (const mode of opts.modes) { const startedAt = Date.now(); const runId = `${commit}-${suite}-${mode}-${opts.seed}`; const record: EvalRunRecord = { - schema_version: 2, + schema_version: 3, run_id: runId, ran_at: new Date().toISOString(), suite: suite as ValidSuite, diff --git a/src/commands/eval.ts b/src/commands/eval.ts index b533f2ce0..8785b40f8 100644 --- a/src/commands/eval.ts +++ b/src/commands/eval.ts @@ -52,6 +52,14 @@ export async function runEvalCommand(engine: BrainEngine, args: string[]): Promi const { runEvalCrossModal } = await import('./eval-cross-modal.ts'); process.exit(await runEvalCrossModal(args.slice(1))); } + if (sub === 'brainbench') { + // No-DB sub-subcommand: brainbench brings its own hermetic PGLite. The + // cli.ts dispatcher routes the user-facing path before connectEngine; + // this branch only fires on re-entry. Engine intentionally unused. The + // command owns its exit codes (0 pass / 1 regression / 2 error). + const { runEvalBrainBench } = await import('./eval-brainbench.ts'); + await runEvalBrainBench(args.slice(1)); + } if (sub === 'code-retrieval') { // v0.33.3 pre-w0 — code-retrieval baseline / gate harness. Needs a brain // for the baseline (BaselineStrategy calls hybridSearch); --compare diff --git a/test/eval-brainbench-e2e.test.ts b/test/eval-brainbench-e2e.test.ts new file mode 100644 index 000000000..a6c7ead07 --- /dev/null +++ b/test/eval-brainbench-e2e.test.ts @@ -0,0 +1,214 @@ +/** + * BrainBench CLI e2e — subprocess runs against a SMALL tmp corpus so the + * literal exit codes (the CI product, decision 9) are asserted end-to-end: + * 0 pass · 1 regression · 2 error/inconclusive. Also pins: the --out artifact + * is complete valid JSON with the _meta.metric_glossary block, --update-baseline + * is byte-deterministic across runs, anti-vacuous-pass, and the run-all wiring + * (full corpus, in-process). + */ +import { beforeAll, describe, expect, test } from 'bun:test'; +import { cpSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync, existsSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +const REPO = process.cwd(); +let root: string; +let fixtures: string; +let gold: string; + +function run(args: string[], cwd = REPO): { exitCode: number; stdout: string; stderr: string } { + const proc = Bun.spawnSync(['bun', 'src/cli.ts', 'eval', 'brainbench', ...args], { + cwd, + env: { ...process.env, GBRAIN_QUIET: '1' }, + stdout: 'pipe', + stderr: 'pipe', + }); + return { + exitCode: proc.exitCode ?? -1, + stdout: proc.stdout.toString(), + stderr: proc.stderr.toString(), + }; +} + +beforeAll(() => { + root = mkdtempSync(join(tmpdir(), 'bb-e2e-')); + fixtures = join(root, 'fixtures'); + gold = join(root, 'gold'); + mkdirSync(fixtures, { recursive: true }); + mkdirSync(gold, { recursive: true }); + for (const id of ['kta-001-deal-recall', 'kta-002-quiet-smalltalk']) { + cpSync(join(REPO, 'evals/brainbench/fixtures', `${id}.fixture.json`), join(fixtures, `${id}.fixture.json`)); + cpSync(join(REPO, 'evals/brainbench/gold', `${id}.gold.json`), join(gold, `${id}.gold.json`)); + } +}); + +describe('exit contract over a multi-brain run (PGLite exitCode-hijack guard)', () => { + test('clean run: exit 0, --out is complete valid JSON with the glossary block', () => { + const out = join(root, 'r1.json'); + const r = run(['--fixtures', fixtures, '--gold', gold, '--harness', 'openclaw', '--out', out]); + expect(r.exitCode).toBe(0); + const doc = JSON.parse(readFileSync(out, 'utf-8')); + expect(doc.receipt.result_schema_version).toBe(1); + expect(doc.cells.length).toBeGreaterThan(0); + expect(doc._meta.metric_glossary.know_to_ask_failure_rate).toContain('thesis failure mode'); + expect(doc.seed_failures).toEqual([]); + expect(r.stdout).toContain('# BrainBench scoreboard'); + }, 30_000); + + test('--update-baseline is byte-deterministic across two runs (decision 10)', () => { + const b1 = join(root, 'base1.json'); + const b2 = join(root, 'base2.json'); + expect(run(['--fixtures', fixtures, '--gold', gold, '--update-baseline', b1]).exitCode).toBe(0); + expect(run(['--fixtures', fixtures, '--gold', gold, '--update-baseline', b2]).exitCode).toBe(0); + expect(readFileSync(b1, 'utf-8')).toBe(readFileSync(b2, 'utf-8')); + }, 60_000); + + test('--compare against own baseline: exit 0 PASS', () => { + const b = join(root, 'base1.json'); + const r = run(['--fixtures', fixtures, '--gold', gold, '--compare', b]); + expect(r.exitCode).toBe(0); + expect(r.stdout).toContain('## Gate: PASS (same-hash)'); + }, 30_000); + + test('doctored main baseline (pretends fewer failures): exit 1 REGRESSION with named breach', () => { + const doctored = JSON.parse(readFileSync(join(root, 'base1.json'), 'utf-8')); + const cellKey = Object.keys(doctored.counts).find((k) => doctored.counts[k].gold_total > 0)!; + doctored.counts[cellKey].gold_failed = Math.max(0, doctored.counts[cellKey].gold_failed - 1); + // also make the run's count strictly greater by raising the bar impossibly + doctored.counts[cellKey].gold_failed = -1 as never; + const path = join(root, 'doctored.json'); + writeFileSync(path, JSON.stringify(doctored, null, 2)); + const r = run(['--fixtures', fixtures, '--gold', gold, '--compare', path]); + expect(r.exitCode).toBe(1); + expect(r.stdout).toContain('## Gate: REGRESSION'); + expect(r.stdout).toContain('newly-failed'); + }, 30_000); + + test('--allow-regression flips the same comparison to exit 0 and records the reason', () => { + const r = run([ + '--fixtures', fixtures, '--gold', gold, + '--compare', join(root, 'doctored.json'), + '--allow-regression', 'e2e test bless', + ]); + expect(r.exitCode).toBe(0); + expect(r.stdout).toContain('regression allowed: e2e test bless'); + }, 30_000); + + test('fixtures_hash mismatch without a committed baseline: exit 2 INCONCLUSIVE', () => { + const foreign = JSON.parse(readFileSync(join(root, 'base1.json'), 'utf-8')); + foreign.fixtures_hash = 'f'.repeat(64); + const path = join(root, 'foreign.json'); + writeFileSync(path, JSON.stringify(foreign, null, 2)); + const r = run([ + '--fixtures', fixtures, '--gold', gold, + '--compare', path, + '--committed-baseline', join(root, 'nonexistent.json'), + ]); + expect(r.exitCode).toBe(2); + expect(r.stdout).toContain('corpus-bless'); + }, 30_000); +}); + +describe('anti-vacuous-pass + error paths (always exit 2, never 0)', () => { + test('empty fixtures dir: exit 2', () => { + const empty = join(root, 'empty-fixtures'); + const emptyGold = join(root, 'empty-gold'); + mkdirSync(empty, { recursive: true }); + mkdirSync(emptyGold, { recursive: true }); + const r = run(['--fixtures', empty, '--gold', emptyGold]); + expect(r.exitCode).toBe(2); + expect(r.stderr).toContain('vacuous'); + }, 30_000); + + test('suite filter matching zero fixtures: exit 2', () => { + const r = run(['--fixtures', fixtures, '--gold', gold, '--suite', 'continuity']); + expect(r.exitCode).toBe(2); + expect(r.stderr).toContain('vacuous'); + }, 30_000); + + test('malformed fixture JSON: exit 2 with the validation error named', () => { + const badRoot = mkdtempSync(join(tmpdir(), 'bb-bad-')); + const badF = join(badRoot, 'fixtures'); + const badG = join(badRoot, 'gold'); + mkdirSync(badF); + mkdirSync(badG); + writeFileSync(join(badF, 'bad.fixture.json'), '{ not json'); + const r = run(['--fixtures', badF, '--gold', badG]); + expect(r.exitCode).toBe(2); + expect(r.stderr).toContain('invalid JSON'); + }, 30_000); + + test('usage error (unknown flag): exit 2 with usage', () => { + const r = run(['--frobnicate']); + expect(r.exitCode).toBe(2); + expect(r.stderr).toContain('Usage: gbrain eval brainbench'); + }, 30_000); + + test('seed failure (duplicate slug across seed pages in one source): exit 2, fixture named', () => { + const dupRoot = mkdtempSync(join(tmpdir(), 'bb-dup-')); + const dupF = join(dupRoot, 'fixtures'); + const dupG = join(dupRoot, 'gold'); + mkdirSync(dupF); + mkdirSync(dupG); + // A page whose content exceeds importFromContent's size cap → status 'skipped' → SeedError. + const huge = 'x'.repeat(5_000_001); + writeFileSync( + join(dupF, 'seedfail-001.fixture.json'), + JSON.stringify({ + schema_version: 1, + fixture_id: 'seedfail-001', + suites: ['know-to-ask'], + seed_pages: [{ slug: 'people/too-big', content: `---\ntitle: Too Big\n---\n${huge}` }], + turns: [{ turn_id: 1, role: 'user', text: 'Hello Too Big' }], + }), + ); + writeFileSync( + join(dupG, 'seedfail-001.gold.json'), + JSON.stringify({ fixture_id: 'seedfail-001', turns: { '1': { should_retrieve: false } } }), + ); + const r = run(['--fixtures', dupF, '--gold', dupG]); + expect(r.exitCode).toBe(2); + expect(r.stderr).toContain('SEED FAILURES'); + expect(r.stderr).toContain('seedfail-001'); + }, 30_000); +}); + +describe('holdout discipline (decision 22)', () => { + test('gate mode excludes holdout fixtures; --include-holdout scores them', async () => { + const { loadCorpus } = await import('../src/eval/brainbench/fixtures.ts'); + const { runBrainBench } = await import('../src/eval/brainbench/harness.ts'); + const corpus = await loadCorpus('evals/brainbench/fixtures', 'evals/brainbench/gold'); + const holdoutIds = new Set( + corpus.fixtures.filter((f) => f.fixture.holdout).map((f) => f.fixture.fixture_id), + ); + expect(holdoutIds.size).toBeGreaterThan(0); + const pick = corpus.fixtures + .filter((f) => f.fixture.category === 'kta-pos') + .filter((f, i, arr) => f.fixture.holdout || arr.findIndex((x) => !x.fixture.holdout) === i) + .slice(0, 4); + const sub = { ...corpus, fixtures: pick }; + const gateRun = await runBrainBench(sub, { + harnesses: ['openclaw'], suites: ['know-to-ask'], includeHoldout: false, llm: false, + }); + for (const r of gateRun.turn_rows) expect(holdoutIds.has(r.fixture_id)).toBe(false); + const pubRun = await runBrainBench(sub, { + harnesses: ['openclaw'], suites: ['know-to-ask'], includeHoldout: true, llm: false, + }); + expect(pubRun.turn_rows.length).toBeGreaterThan(gateRun.turn_rows.length); + }, 60_000); +}); + +describe('run-all wiring (decision 16) — full corpus, in-process', () => { + test('runBrainBenchCore completes over the committed corpus with 12 cells', async () => { + const { runBrainBenchCore } = await import('../src/commands/eval-brainbench.ts'); + const core = await runBrainBenchCore(); + expect(core.status).toBe('completed'); + expect(Object.keys(core.cells ?? {}).length).toBe(12); + expect(core.fixtures_hash).toBeDefined(); + // committed baseline matches the committed corpus hash (drift guard) + if (existsSync('evals/brainbench/baselines/main.json')) { + const baseline = JSON.parse(readFileSync('evals/brainbench/baselines/main.json', 'utf-8')); + expect(baseline.fixtures_hash).toBe(core.fixtures_hash); + } + }, 120_000); +}); diff --git a/test/eval-run-all.test.ts b/test/eval-run-all.test.ts index c12ef15d7..061df693f 100644 --- a/test/eval-run-all.test.ts +++ b/test/eval-run-all.test.ts @@ -155,7 +155,7 @@ describe('persistRunRecord audit trail', () => { test('appends to eval-results.jsonl, creates dir if missing', () => { const record: EvalRunRecord = { - schema_version: 2, + schema_version: 3, run_id: 'abc123-longmemeval-conservative-42', ran_at: '2026-05-12T12:00:00Z', suite: 'longmemeval', @@ -173,12 +173,12 @@ describe('persistRunRecord audit trail', () => { expect(content).toContain('abc123-longmemeval-conservative-42'); const parsed = JSON.parse(content.trim()); expect(parsed.mode).toBe('conservative'); - expect(parsed.schema_version).toBe(2); + expect(parsed.schema_version).toBe(3); }); test('appends multiple records (NDJSON)', () => { const base = { - schema_version: 2 as const, + schema_version: 3 as const, ran_at: '2026-05-12T12:00:00Z', suite: 'longmemeval' as const, commit: 'abc', @@ -194,4 +194,24 @@ describe('persistRunRecord audit trail', () => { const lines = content.trim().split('\n'); expect(lines.length).toBe(3); }); + + test("v3: brainbench records carry mode 'n/a' (decision 16 — no params.mode_independent hack)", () => { + const record: EvalRunRecord = { + schema_version: 3, + run_id: 'abc123-brainbench-na-42', + ran_at: '2026-06-12T12:00:00Z', + suite: 'brainbench', + mode: 'n/a', + commit: 'abc123', + seed: 42, + params: { fixtures_hash: 'deadbeef', cells: {} }, + status: 'completed', + duration_ms: 1, + }; + persistRunRecord(tmp, record, tmp); + const parsed = JSON.parse(readFileSync(join(tmp, 'eval-results.jsonl'), 'utf-8').trim()); + expect(parsed.mode).toBe('n/a'); + expect(parsed.suite).toBe('brainbench'); + expect(parsed.params.mode_independent).toBeUndefined(); + }); });