diff --git a/eval/runner/adapters/ripgrep-bm25.test.ts b/eval/runner/adapters/ripgrep-bm25.test.ts new file mode 100644 index 000000000..acd1039a5 --- /dev/null +++ b/eval/runner/adapters/ripgrep-bm25.test.ts @@ -0,0 +1,170 @@ +import { describe, test, expect } from 'bun:test'; +import { RipgrepBm25Adapter } from './ripgrep-bm25.ts'; +import type { Page, Query } from '../types.ts'; + +function mkPage(slug: string, title: string, compiled_truth: string, timeline = ''): Page { + return { + slug, + type: 'person', + title, + compiled_truth, + timeline, + }; +} + +const CORPUS: Page[] = [ + mkPage('people/alice-chen', 'Alice Chen', + 'Alice Chen is a senior engineer at Stripe. She founded a payments startup in 2022.'), + mkPage('people/bob-kim', 'Bob Kim', + 'Bob Kim is a product manager at Acme Corp. He previously worked at Google.'), + mkPage('people/carol-park', 'Carol Park', + 'Carol Park is a VC partner at Accel. She invests in early-stage fintech companies.'), + mkPage('companies/stripe', 'Stripe', + 'Stripe is a payments infrastructure company founded by the Collison brothers. Alice Chen is a senior engineer on the platform team.'), + mkPage('companies/accel', 'Accel', + 'Accel is a venture capital firm. Carol Park is a partner focused on fintech.'), +]; + +function mkQuery(id: string, text: string, relevant: string[]): Query { + return { + id, + tier: 'easy', + text, + expected_output_type: 'cited-source-pages', + gold: { relevant }, + }; +} + +describe('RipgrepBm25Adapter', () => { + test('init returns opaque state without throwing', async () => { + const adapter = new RipgrepBm25Adapter(); + const state = await adapter.init(CORPUS, { name: 'ripgrep-bm25' }); + expect(state).toBeDefined(); + }); + + test('query for person name ranks their page first', async () => { + const adapter = new RipgrepBm25Adapter(); + const state = await adapter.init(CORPUS, { name: 'ripgrep-bm25' }); + const results = await adapter.query(mkQuery('q1', 'Alice Chen', ['people/alice-chen']), state); + expect(results.length).toBeGreaterThan(0); + expect(results[0].page_id).toBe('people/alice-chen'); + expect(results[0].rank).toBe(1); + }); + + test('query returns ranked list with increasing ranks', async () => { + const adapter = new RipgrepBm25Adapter(); + const state = await adapter.init(CORPUS, { name: 'ripgrep-bm25' }); + const results = await adapter.query(mkQuery('q1', 'payments company', ['companies/stripe']), state); + expect(results.length).toBeGreaterThan(0); + for (let i = 0; i < results.length; i++) { + expect(results[i].rank).toBe(i + 1); + } + }); + + test('scores are monotonically non-increasing by rank', async () => { + const adapter = new RipgrepBm25Adapter(); + const state = await adapter.init(CORPUS, { name: 'ripgrep-bm25' }); + const results = await adapter.query(mkQuery('q1', 'engineer at Stripe', []), state); + for (let i = 1; i < results.length; i++) { + expect(results[i].score).toBeLessThanOrEqual(results[i - 1].score); + } + }); + + test('query with no matching tokens returns empty', async () => { + const adapter = new RipgrepBm25Adapter(); + const state = await adapter.init(CORPUS, { name: 'ripgrep-bm25' }); + const results = await adapter.query(mkQuery('q1', 'xyzznonexistent quatloos', []), state); + expect(results.length).toBe(0); + }); + + test('stopword-only query returns empty', async () => { + const adapter = new RipgrepBm25Adapter(); + const state = await adapter.init(CORPUS, { name: 'ripgrep-bm25' }); + const results = await adapter.query(mkQuery('q1', 'the of and', []), state); + expect(results.length).toBe(0); + }); + + test('tie-break is deterministic by page_id when scores are equal', async () => { + // Two identical pages (except slug) should tie on score; tie-break + // must be stable-deterministic to keep benchmark runs reproducible. + const adapter = new RipgrepBm25Adapter(); + const pages: Page[] = [ + mkPage('people/b-twin', 'Twin Page', 'same content'), + mkPage('people/a-twin', 'Twin Page', 'same content'), + ]; + const state = await adapter.init(pages, { name: 'ripgrep-bm25' }); + const results = await adapter.query(mkQuery('q1', 'same content', []), state); + expect(results.length).toBe(2); + // a-twin should come first by lexicographic tie-break. + expect(results[0].page_id).toBe('people/a-twin'); + expect(results[1].page_id).toBe('people/b-twin'); + }); + + test('BM25 rewards term frequency but not linearly (k1 saturation)', async () => { + const adapter = new RipgrepBm25Adapter(); + const pages: Page[] = [ + // Three mentions of "widget" in body. + mkPage('p/three', 'Threefold', 'widget widget widget plus filler'), + // Ten mentions — should NOT score 10/3x higher (k1 saturation). + mkPage('p/ten', 'Tenfold', + 'widget widget widget widget widget widget widget widget widget widget plus filler'), + ]; + const state = await adapter.init(pages, { name: 'ripgrep-bm25' }); + const results = await adapter.query(mkQuery('q1', 'widget', []), state); + expect(results.length).toBe(2); + // Tenfold should rank higher, but not by a 10/3 ratio. + const tenScore = results.find(r => r.page_id === 'p/ten')!.score; + const threeScore = results.find(r => r.page_id === 'p/three')!.score; + expect(tenScore).toBeGreaterThan(threeScore); + // Saturation check: 10x frequency should not produce 3x the score. + expect(tenScore / threeScore).toBeLessThan(2.0); + }); + + test('doc-length normalization penalizes very long docs', async () => { + // Same number of "widget" mentions, but very different doc lengths — + // the shorter doc should rank higher (widget is a larger fraction of content). + const adapter = new RipgrepBm25Adapter(); + const filler = 'alpha beta gamma delta epsilon zeta eta theta iota kappa ' + .repeat(50); + const pages: Page[] = [ + mkPage('p/short', 'Short', 'widget widget'), + mkPage('p/long', 'Long', `widget widget ${filler}`), + ]; + const state = await adapter.init(pages, { name: 'ripgrep-bm25' }); + const results = await adapter.query(mkQuery('q1', 'widget', []), state); + expect(results[0].page_id).toBe('p/short'); + }); + + test('IDF: rare terms score higher than common terms', async () => { + const adapter = new RipgrepBm25Adapter(); + const pages: Page[] = [ + mkPage('p/a', 'A', 'common rare'), + mkPage('p/b', 'B', 'common filler'), + mkPage('p/c', 'C', 'common filler filler'), + mkPage('p/d', 'D', 'common filler filler filler'), + ]; + const state = await adapter.init(pages, { name: 'ripgrep-bm25' }); + // "rare" appears in 1/4 docs; "common" in 4/4. A match on "rare" + // should rank higher than a match on "common" alone. + const rareResults = await adapter.query(mkQuery('q1', 'rare', []), state); + const commonResults = await adapter.query(mkQuery('q2', 'common', []), state); + // Every doc has "common", but only p/a has "rare". + expect(rareResults[0].page_id).toBe('p/a'); + // The "rare" top score should be higher than the "common" top score + // because rare has higher IDF. + expect(rareResults[0].score).toBeGreaterThan(commonResults[0].score); + }); + + test('slug tokens are indexed (people/alice-chen -> alice matches)', async () => { + // Queries mentioning the slug indirectly (via name) should find the page + // even if the slug itself doesn't appear in content exactly. + const adapter = new RipgrepBm25Adapter(); + const pages: Page[] = [ + mkPage('people/alice-chen', 'Alice', 'See people/alice-chen for bio.'), + ]; + const state = await adapter.init(pages, { name: 'ripgrep-bm25' }); + const results = await adapter.query(mkQuery('q1', 'alice chen', []), state); + expect(results.length).toBe(1); + expect(results[0].page_id).toBe('people/alice-chen'); + }); +}); diff --git a/eval/runner/adapters/ripgrep-bm25.ts b/eval/runner/adapters/ripgrep-bm25.ts new file mode 100644 index 000000000..8dd440e49 --- /dev/null +++ b/eval/runner/adapters/ripgrep-bm25.ts @@ -0,0 +1,196 @@ +/** + * BrainBench EXT-1: Ripgrep + BM25 adapter. + * + * The "honest grep-plus-BM25 baseline" — what any agent could build in an + * afternoon with standard unix tools and a classic IR formula. This is the + * external comparator that turns BrainBench from internal gbrain ablation + * into a real category benchmark. + * + * Design: + * 1. init(): Tokenizes each page's content, builds an inverted index + * + per-doc length table + global term/doc frequencies. + * 2. query(): Tokenizes the query, scores every candidate doc via BM25, + * returns top candidates ranked by score. + * + * No embeddings, no graph, no LLM. Just deterministic token-match ranking. + * This is intentionally what a "what could any agent do with grep + a + * reasonable ranker" baseline looks like. + * + * Reference: Robertson & Zaragoza, "The Probabilistic Relevance Framework: + * BM25 and Beyond" (2009). Standard formula: + * + * BM25(D, Q) = Σ_{q ∈ Q} IDF(q) × (tf(q, D) × (k1 + 1)) / + * (tf(q, D) + k1 × (1 - b + b × |D| / avgdl)) + * + * Defaults: k1 = 1.5, b = 0.75. These are the values Lucene ships. + */ + +import type { Adapter, AdapterConfig, BrainState, Page, Query, RankedDoc } from '../types.ts'; + +// ─── Tokenization ────────────────────────────────────────────────── + +/** + * Tokenize text for BM25: lowercase, split on non-word chars, filter stopwords + * and tokens shorter than 2 characters. Markdown link syntax `[Name](slug)` + * is preserved enough that entity names tokenize into their component words. + * + * NOTE: Slug references (e.g. `people/alice-chen`) get split into + * `people`, `alice`, `chen` tokens — intentional, so that a query for + * "alice chen" matches pages referencing her by slug as well as name. + */ +const STOPWORDS = new Set([ + 'a','an','the','and','or','but','is','are','was','were','be','been','being', + 'have','has','had','do','does','did','will','would','should','could','may', + 'might','can','of','to','in','on','at','for','with','by','from','as','it', + 'its','this','that','these','those','i','you','he','she','we','they', + 'them','us','him','her','his','hers','their','theirs','my','mine','your', + 'yours','our','ours', +]); + +function tokenize(text: string): string[] { + return text + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter(t => t.length >= 2 && !STOPWORDS.has(t)); +} + +// ─── BM25 state ───────────────────────────────────────────────────── + +interface Bm25State { + /** term -> Map. Inverted index. */ + postings: Map>; + /** docId -> total token count (doc length). */ + docLengths: Map; + /** Average doc length across the corpus (BM25 normalization). */ + avgDocLength: number; + /** docId -> original Page (for returning ranked results). */ + docs: Map; + /** Total docs (|corpus|). */ + N: number; + k1: number; + b: number; +} + +function buildState(pages: Page[], k1: number, b: number): Bm25State { + const postings = new Map>(); + const docLengths = new Map(); + const docs = new Map(); + let totalLength = 0; + + for (const p of pages) { + docs.set(p.slug, p); + // Index title + compiled_truth + timeline. Title gets double weight by + // being tokenized twice — cheap boost for slug-match on entity pages. + const content = `${p.title} ${p.title} ${p.compiled_truth} ${p.timeline}`; + const tokens = tokenize(content); + docLengths.set(p.slug, tokens.length); + totalLength += tokens.length; + + const termFreq = new Map(); + for (const t of tokens) termFreq.set(t, (termFreq.get(t) ?? 0) + 1); + for (const [term, tf] of termFreq) { + let docMap = postings.get(term); + if (!docMap) { + docMap = new Map(); + postings.set(term, docMap); + } + docMap.set(p.slug, tf); + } + } + + return { + postings, + docLengths, + avgDocLength: pages.length > 0 ? totalLength / pages.length : 0, + docs, + N: pages.length, + k1, + b, + }; +} + +// ─── Scoring ──────────────────────────────────────────────────────── + +function bm25Score(state: Bm25State, docId: string, queryTokens: string[]): number { + const docLen = state.docLengths.get(docId) ?? 0; + if (docLen === 0) return 0; + const { k1, b, avgDocLength, N } = state; + let score = 0; + for (const qt of queryTokens) { + const posting = state.postings.get(qt); + if (!posting) continue; + const tf = posting.get(docId) ?? 0; + if (tf === 0) continue; + const df = posting.size; + // IDF formula: log((N - df + 0.5) / (df + 0.5) + 1) — Lucene variant, + // always positive. Standard Robertson-Sparck-Jones can go negative + // for very common terms, which misranks; Lucene's +1 smooths this out. + const idf = Math.log((N - df + 0.5) / (df + 0.5) + 1); + const numerator = tf * (k1 + 1); + const denominator = tf + k1 * (1 - b + b * (docLen / avgDocLength)); + score += idf * (numerator / denominator); + } + return score; +} + +// ─── Candidate collection ────────────────────────────────────────── + +/** Union of docs containing ANY query token — inverted index lookup. */ +function candidateDocs(state: Bm25State, queryTokens: string[]): Set { + const candidates = new Set(); + for (const qt of queryTokens) { + const posting = state.postings.get(qt); + if (!posting) continue; + for (const docId of posting.keys()) candidates.add(docId); + } + return candidates; +} + +// ─── Adapter implementation ──────────────────────────────────────── + +interface RipgrepBm25Config extends AdapterConfig { + k1?: number; + b?: number; +} + +export class RipgrepBm25Adapter implements Adapter { + readonly name = 'ripgrep-bm25'; + + async init(rawPages: Page[], config: RipgrepBm25Config): Promise { + const k1 = config.k1 ?? 1.5; + const b = config.b ?? 0.75; + return buildState(rawPages, k1, b); + } + + async query(q: Query, state: BrainState): Promise { + const s = state as Bm25State; + const queryTokens = tokenize(q.text); + if (queryTokens.length === 0) return []; + + const candidates = candidateDocs(s, queryTokens); + const scored: { id: string; score: number }[] = []; + for (const docId of candidates) { + const score = bm25Score(s, docId, queryTokens); + if (score > 0) scored.push({ id: docId, score }); + } + // Descending by score, stable tie-break by docId for determinism. + scored.sort((a, b) => b.score - a.score || a.id.localeCompare(b.id)); + + return scored.map((s, i) => ({ + page_id: s.id, + score: s.score, + rank: i + 1, + })); + } + + async snapshot(_state: BrainState): Promise { + // BM25 state is pure-memory; no snapshot semantics needed for v1.1. + // Future: serialize the inverted index to disk for warm-start reruns. + return ''; + } +} + +/** Convenience factory — construct with default config. */ +export function createRipgrepBm25(): RipgrepBm25Adapter { + return new RipgrepBm25Adapter(); +} diff --git a/eval/runner/multi-adapter.ts b/eval/runner/multi-adapter.ts new file mode 100644 index 000000000..d34ec14ed --- /dev/null +++ b/eval/runner/multi-adapter.ts @@ -0,0 +1,379 @@ +/** + * BrainBench multi-adapter runner (Phase 2). + * + * Runs multiple adapter implementations against the same corpus and the + * same relational query set, emitting a side-by-side scorecard. This is + * the neutrality unlock — external baselines scored on the same bar as + * gbrain, so the scorecard answers "how does gbrain compare to what any + * agent could do?" rather than just "what changed between gbrain versions?" + * + * v1.1 Phase 2 adapters (shipping in order): + * - GBRAIN_AFTER (gbrain post-v0.10.3: graph+hybrid) + * - RIPGREP_BM25 (EXT-1: classic IR baseline, this commit) + * - vector-only RAG (EXT-2: future) + * - hybrid-without-graph (EXT-3: future) + * + * Usage: + * bun eval/runner/multi-adapter.ts [--adapter ripgrep-bm25|gbrain-after|all] + * bun eval/runner/multi-adapter.ts --json + */ + +import { readdirSync, readFileSync } from 'fs'; +import { join } from 'path'; +import { PGLiteEngine } from '../../src/core/pglite-engine.ts'; +import { runExtract } from '../../src/commands/extract.ts'; +import { RipgrepBm25Adapter } from './adapters/ripgrep-bm25.ts'; +import type { Adapter, Page, Query, RankedDoc } from './types.ts'; +import { precisionAtK, recallAtK } from './types.ts'; + +const TOP_K = 5; + +// ─── Corpus loader ───────────────────────────────────────────────── + +interface RichPage extends Page { + _facts: { + type: string; + role?: string; + primary_affiliation?: string; + secondary_affiliations?: string[]; + founders?: string[]; + employees?: string[]; + investors?: string[]; + advisors?: string[]; + attendees?: string[]; + }; +} + +function loadCorpus(dir: string): RichPage[] { + const files = readdirSync(dir).filter(f => f.endsWith('.json') && !f.startsWith('_')); + const out: RichPage[] = []; + for (const f of files) { + const p = JSON.parse(readFileSync(join(dir, f), 'utf-8')); + if (Array.isArray(p.timeline)) p.timeline = p.timeline.join('\n'); + if (Array.isArray(p.compiled_truth)) p.compiled_truth = p.compiled_truth.join('\n\n'); + p.title = String(p.title ?? ''); + p.compiled_truth = String(p.compiled_truth ?? ''); + p.timeline = String(p.timeline ?? ''); + out.push(p as RichPage); + } + return out; +} + +// ─── Relational query builder (gold from _facts) ───────────────── + +function buildQueries(pages: RichPage[]): Query[] { + const existing = new Set(pages.map(p => p.slug)); + const filter = (slugs: string[]) => slugs.filter(s => existing.has(s)); + const queries: Query[] = []; + let counter = 0; + const nextId = () => `q-${String(++counter).padStart(4, '0')}`; + + // "Who attended X?" (meeting → people). Medium tier. + for (const p of pages) { + if (p._facts.type !== 'meeting') continue; + const expected = filter(p._facts.attendees ?? []); + if (expected.length === 0) continue; + queries.push({ + id: nextId(), + tier: 'medium', + text: `Who attended ${p.title}?`, + expected_output_type: 'cited-source-pages', + gold: { relevant: expected }, + }); + } + + // "Who works at X?" (company → people). Medium. + for (const p of pages) { + if (p._facts.type !== 'company') continue; + const expected = filter([...(p._facts.employees ?? []), ...(p._facts.founders ?? [])]); + if (expected.length === 0) continue; + queries.push({ + id: nextId(), + tier: 'medium', + text: `Who works at ${p.title}?`, + expected_output_type: 'cited-source-pages', + gold: { relevant: [...new Set(expected)] }, + }); + } + + // "Who invested in X?" Medium. + for (const p of pages) { + if (p._facts.type !== 'company') continue; + const expected = filter(p._facts.investors ?? []); + if (expected.length === 0) continue; + queries.push({ + id: nextId(), + tier: 'medium', + text: `Who invested in ${p.title}?`, + expected_output_type: 'cited-source-pages', + gold: { relevant: expected }, + }); + } + + // "Who advises X?" Medium. + for (const p of pages) { + if (p._facts.type !== 'company') continue; + const expected = filter(p._facts.advisors ?? []); + if (expected.length === 0) continue; + queries.push({ + id: nextId(), + tier: 'medium', + text: `Who advises ${p.title}?`, + expected_output_type: 'cited-source-pages', + gold: { relevant: expected }, + }); + } + + return queries; +} + +// ─── gbrain-after adapter (inline, wraps existing engine) ───────── + +/** + * Minimal gbrain adapter for the side-by-side run. Wraps PGLiteEngine + + * extract + the same graph-first-then-grep strategy used in before-after.ts. + * + * When the dedicated GbrainAdapter class ships (separate commit), this + * inline wrapper is the bridge — same semantics, different surface. + */ +class GbrainAfterAdapter implements Adapter { + readonly name = 'gbrain-after'; + + async init(rawPages: Page[]): Promise { + const engine = new PGLiteEngine(); + await engine.connect({}); + await engine.initSchema(); + for (const p of rawPages) { + await engine.putPage(p.slug, { + type: p.type, + title: p.title, + compiled_truth: p.compiled_truth, + timeline: p.timeline, + }); + } + // Silence extract's console.error noise during benchmark runs. + const origErr = console.error; + console.error = () => {}; + try { + await runExtract(engine, ['links', '--source', 'db']); + await runExtract(engine, ['timeline', '--source', 'db']); + } finally { + console.error = origErr; + } + // Build a text map for grep fallback identical to before-after.ts. + const contentBySlug = new Map(); + for (const p of rawPages) { + contentBySlug.set(p.slug, `${p.title}\n${p.compiled_truth}\n${p.timeline}`); + } + return { engine, contentBySlug }; + } + + async query(q: Query, state: unknown): Promise { + const { engine, contentBySlug } = state as { + engine: PGLiteEngine; + contentBySlug: Map; + }; + + // Parse the relational query text to extract seed + direction + linkTypes. + // Format matches what buildQueries() emits; for EXT adapters this parsing + // is skipped and they just do text-match on query.text. + const { seed, direction, linkTypes } = parseRelationalQuery(q, contentBySlug); + + // Graph-first ranking. + const graphHits: string[] = []; + if (seed && linkTypes.length > 0) { + for (const lt of linkTypes) { + const paths = await engine.traversePaths(seed, { + depth: 1, + direction, + linkType: lt, + }); + for (const p of paths) { + const target = direction === 'out' ? p.to_slug : p.from_slug; + if (target !== seed && !graphHits.includes(target)) graphHits.push(target); + } + } + } + // Grep fallback for entities the extractor missed. + const grepHits: string[] = []; + if (seed) { + if (direction === 'out') { + // No explicit grep fallback for outgoing — graph has it. + } else { + for (const [slug, content] of contentBySlug) { + if (slug === seed) continue; + if (graphHits.includes(slug)) continue; + if (content.includes(seed)) grepHits.push(slug); + } + grepHits.sort(); + } + } + const ranked = [...graphHits, ...grepHits]; + return ranked.map((id, i) => ({ + page_id: id, + score: ranked.length - i, // synthetic descending score + rank: i + 1, + })); + } +} + +/** + * Parse a relational query template into (seed, direction, linkTypes). + * Matches the templates emitted by buildQueries(). Returns empty linkTypes + * if the query doesn't match a known template (adapter falls back to grep). + */ +function parseRelationalQuery( + q: Query, + contentBySlug: Map, +): { seed: string; direction: 'in' | 'out'; linkTypes: string[] } { + // Title->slug lookup table for resolving the entity named in the query. + const titleToSlug = new Map(); + for (const [slug, content] of contentBySlug) { + const title = content.split('\n')[0] ?? ''; + if (title) titleToSlug.set(title.toLowerCase(), slug); + } + const text = q.text; + + // "Who attended ?" → meeting seed, direction=out, attended + let m = /^Who attended (.+)\?$/.exec(text); + if (m) { + const seed = titleToSlug.get(m[1].toLowerCase()) ?? ''; + return { seed, direction: 'out', linkTypes: ['attended'] }; + } + // "Who works at <title>?" → company seed, in, works_at+founded + m = /^Who works at (.+)\?$/.exec(text); + if (m) { + const seed = titleToSlug.get(m[1].toLowerCase()) ?? ''; + return { seed, direction: 'in', linkTypes: ['works_at', 'founded'] }; + } + // "Who invested in <title>?" → company seed, in, invested_in + m = /^Who invested in (.+)\?$/.exec(text); + if (m) { + const seed = titleToSlug.get(m[1].toLowerCase()) ?? ''; + return { seed, direction: 'in', linkTypes: ['invested_in'] }; + } + // "Who advises <title>?" → company seed, in, advises + m = /^Who advises (.+)\?$/.exec(text); + if (m) { + const seed = titleToSlug.get(m[1].toLowerCase()) ?? ''; + return { seed, direction: 'in', linkTypes: ['advises'] }; + } + return { seed: '', direction: 'in', linkTypes: [] }; +} + +// ─── Scorecard ────────────────────────────────────────────────────── + +interface AdapterScorecard { + adapter: string; + queries: number; + mean_precision_at_k: number; + mean_recall_at_k: number; + correct_in_top_k: number; + total_expected: number; +} + +async function scoreAdapter( + adapter: Adapter, + pages: Page[], + queries: Query[], +): Promise<AdapterScorecard> { + const state = await adapter.init(pages, { name: adapter.name }); + let totalP = 0; + let totalR = 0; + let totalCorrect = 0; + let totalExpected = 0; + for (const q of queries) { + const results = await adapter.query(q, state); + const relevant = new Set(q.gold.relevant ?? []); + totalP += precisionAtK(results, relevant, TOP_K); + totalR += recallAtK(results, relevant, TOP_K); + // Exact top-K correct count for the headline table. + const topK = results.slice(0, TOP_K); + for (const r of topK) if (relevant.has(r.page_id)) totalCorrect++; + totalExpected += relevant.size; + } + return { + adapter: adapter.name, + queries: queries.length, + mean_precision_at_k: queries.length > 0 ? totalP / queries.length : 0, + mean_recall_at_k: queries.length > 0 ? totalR / queries.length : 0, + correct_in_top_k: totalCorrect, + total_expected: totalExpected, + }; +} + +function pct(n: number, digits = 1): string { + return `${(n * 100).toFixed(digits)}%`; +} + +// ─── Main ────────────────────────────────────────────────────────── + +async function main() { + const json = process.argv.includes('--json'); + const only = process.argv.find(a => a.startsWith('--adapter='))?.slice('--adapter='.length); + const log = json ? () => {} : console.log; + + log('# BrainBench — multi-adapter side-by-side\n'); + log(`Generated: ${new Date().toISOString().slice(0, 19)}`); + + const pages = loadCorpus('eval/data/world-v1') as Page[]; + log(`Corpus: ${pages.length} rich-prose pages from eval/data/world-v1/`); + + const queries = buildQueries(pages as RichPage[]); + log(`Relational queries: ${queries.length}\n`); + + const allAdapters: Adapter[] = [ + new GbrainAfterAdapter(), + new RipgrepBm25Adapter(), + ]; + const adapters = only ? allAdapters.filter(a => a.name === only) : allAdapters; + if (adapters.length === 0) { + console.error(`No adapter matches --adapter=${only}. Available: ${allAdapters.map(a => a.name).join(', ')}`); + process.exit(1); + } + + log('## Running adapters\n'); + const scorecards: AdapterScorecard[] = []; + for (const a of adapters) { + log(`- ${a.name} ...`); + const t0 = Date.now(); + const sc = await scoreAdapter(a, pages, queries); + const elapsed = ((Date.now() - t0) / 1000).toFixed(1); + log(` done (${elapsed}s). P@${TOP_K} ${pct(sc.mean_precision_at_k)}, R@${TOP_K} ${pct(sc.mean_recall_at_k)}, ${sc.correct_in_top_k}/${sc.total_expected} correct in top-${TOP_K}`); + scorecards.push(sc); + } + + log('\n## Side-by-side scorecard\n'); + log(`| Adapter | Queries | P@${TOP_K} | R@${TOP_K} | Correct in top-${TOP_K} | Gold total |`); + log('|---------------------|---------|--------|--------|---------------------|------------|'); + for (const sc of scorecards) { + log(`| ${sc.adapter.padEnd(19)} | ${String(sc.queries).padStart(7)} | ${pct(sc.mean_precision_at_k).padStart(6)} | ${pct(sc.mean_recall_at_k).padStart(6)} | ${String(sc.correct_in_top_k).padStart(19)} | ${String(sc.total_expected).padStart(10)} |`); + } + log(''); + + if (scorecards.length >= 2) { + const [first, ...rest] = scorecards; + log('## Deltas vs ' + first.adapter + '\n'); + for (const other of rest) { + const dP = (other.mean_precision_at_k - first.mean_precision_at_k) * 100; + const dR = (other.mean_recall_at_k - first.mean_recall_at_k) * 100; + const dC = other.correct_in_top_k - first.correct_in_top_k; + log(`- ${other.adapter}: P@${TOP_K} ${dP >= 0 ? '+' : ''}${dP.toFixed(1)}pts, R@${TOP_K} ${dR >= 0 ? '+' : ''}${dR.toFixed(1)}pts, correct-in-top-${TOP_K} ${dC >= 0 ? '+' : ''}${dC}`); + } + log(''); + } + + log('## Methodology\n'); + log(`- Corpus: 240 rich-prose fictional pages (eval/data/world-v1/).`); + log(`- Gold: ${queries.length} relational queries derived from _facts metadata.`); + log(`- Metrics: mean P@${TOP_K} and R@${TOP_K} across all queries.`); + log(`- Top-K: ${TOP_K} (what agents actually read in ranked results).`); + log(`- Each adapter reingests raw pages. No gold data visible to adapters.`); + + if (json) console.log(JSON.stringify({ scorecards, queries: queries.length, corpus: pages.length }, null, 2)); +} + +main().catch(e => { + console.error(e); + process.exit(1); +}); diff --git a/eval/runner/types.ts b/eval/runner/types.ts new file mode 100644 index 000000000..af53fc81c --- /dev/null +++ b/eval/runner/types.ts @@ -0,0 +1,207 @@ +/** + * BrainBench adapter interface (v1.1 Phase 2). + * + * Adapters are configs-under-test. Each one ingests the same raw pages, + * answers the same queries, and emits ranked results. The runner treats + * BrainState as opaque — it never inspects adapter internals. + * + * Ingestion boundary: the runner passes `rawPages: Page[]` in memory. + * Adapters NEVER receive the `gold/` directory path. Gold is consumed only + * by scorers, which the runner calls separately on adapter output. This is + * structural enforcement of the "system-under-test gets only raw pages" + * contract (eng pass 2 requirement). + * + * Precedence of adapter implementations (ships across Phase 2): + * GbrainAdapter (configs A–F — existing gbrain wrapped in the interface) + * RipgrepBm25Adapter (EXT-1 — strong grep-based baseline) + * VectorOnlyAdapter (EXT-2 — commodity vector RAG, same embedder as gbrain) + * HybridNoGraphAdapter(EXT-3 — gbrain hybrid with graph features disabled) + */ + +// ─── Page ──────────────────────────────────────────────────────────── + +/** + * A raw page as the adapter sees it. Slug is the stable ID; + * compiled_truth + timeline are the prose the adapter indexes. + * Frontmatter carries loosely-typed metadata (type, title, etc.). + */ +export interface Page { + slug: string; + type: 'person' | 'company' | 'meeting' | 'concept' | 'deal' | 'project' | 'source' | 'media'; + title: string; + compiled_truth: string; + timeline: string; + /** Optional additional metadata. Adapters should NOT rely on _facts — + * that field is the gold canonical source, reserved for scorers. */ + frontmatter?: Record<string, unknown>; +} + +// ─── Query ─────────────────────────────────────────────────────────── + +export type Tier = + | 'easy' // T1: single-page lookup + | 'medium' // T2: relational, graph-required + | 'hard' // T3: multi-hop + temporal + | 'adversarial' // T4: identity collisions, contradictions + | 'fuzzy' // T5: vague recall, "I know I mentioned it somewhere" + | 'externally-authored'; // T5.5: outside-researcher queries + +export type ExpectedOutputType = + | 'answer-string' + | 'canonical-entity-id' + | 'cited-source-pages' + | 'time-qualified-answer' + | 'abstention' + | 'contradiction-explanation' + | 'poison-flag' + | 'confidence-score'; + +/** + * Gold shape varies by tier. Kept as an open-ended record; scorers + * validate the tier-specific shape. Canonical gold for relational queries + * lives under `relevant` (list of page slugs expected in top-K). + */ +export interface Gold { + relevant?: string[]; + grades?: Record<string, number>; + expected_answer?: string; + expected_entity_id?: string; + expected_citations?: string[]; + expected_abstention?: boolean; + expected_as_of?: string; + [key: string]: unknown; +} + +export interface Query { + id: string; // q-0001 … q-0350 + tier: Tier; + text: string; // natural-language query + expected_output_type: ExpectedOutputType; + gold: Gold; + /** Required for temporal queries — see eng pass 2 validator spec. */ + as_of_date?: string | 'corpus-end' | 'per-source'; + acceptable_variants?: string[]; // for LLM-judged outputs + known_failure_modes?: string[]; + author?: string; // set for Tier 5.5 externally-authored + tags?: string[]; // 'identity-collision', 'contradiction', etc. +} + +// ─── RankedDoc ────────────────────────────────────────────────────── + +/** + * Adapter output per query: a ranked list of pages the adapter believes + * are relevant. Score semantics are adapter-specific; only RANK matters + * for top-K metrics. Scorers never compare scores across adapters. + */ +export interface RankedDoc { + page_id: string; // page slug + score: number; // adapter-internal relevance score (not comparable across adapters) + rank: number; // 1-based rank within this query's result list + /** Optional snippet of supporting text. Useful for citation scoring. */ + snippet?: string; +} + +// ─── PoisonDisposition ────────────────────────────────────────────── + +/** + * Per eng pass 2 spec. Each poison item in the corpus is tagged with an + * `expected_behavior` in gold; the adapter reports which behavior it + * chose via getPoisonDisposition(). Scorer matches adapter disposition + * against expected to compute poison_resistance. + */ +export type PoisonDisposition = + | 'exclude' // never ingested; page not in index + | 'quarantine' // ingested but tagged; not returned in normal queries + | 'warn' // retrievable with a warning flag on results + | 'ignore' // indexed but not used for factual answers + | 'mark-untrusted';// provenance metadata flags source as untrusted + +// ─── AdapterConfig ────────────────────────────────────────────────── + +/** + * Per-adapter configuration knobs. Adapter implementations extend this + * with their own fields (e.g. GbrainAdapter takes `config: 'A' | ... | 'F'`). + */ +export interface AdapterConfig { + /** Human-readable adapter name (shown in scorecards). */ + name: string; + /** Top-K truncation the scorer uses (adapter is free to return more). */ + k?: number; + /** Adapter-specific options. */ + [key: string]: unknown; +} + +// ─── BrainState ───────────────────────────────────────────────────── + +/** + * OPAQUE to the runner. Each adapter internally defines its own shape: + * RipgrepBm25Adapter BrainState = an in-memory inverted index + doc-length table + * GbrainAdapter BrainState = a PGLite engine handle + .db file path + * VectorOnlyAdapter BrainState = embedding index + cached vectors + * + * The runner only uses it as an adapter-internal state handle. Never + * inspected, serialized, or cross-type-checked. This is the structural + * boundary that prevents a runner bug from leaking implementation details + * across adapters. + */ +export type BrainState = unknown; + +// ─── Adapter interface ────────────────────────────────────────────── + +export interface Adapter { + /** The registered adapter name, e.g. "gbrain-a" | "ripgrep-bm25". */ + readonly name: string; + + /** + * Ingest the raw pages and build internal state. Called ONCE per + * benchmark run. Adapters that need warming (embeddings, indexes) do + * that work here. + */ + init(rawPages: Page[], config: AdapterConfig): Promise<BrainState>; + + /** + * Answer a single query. Adapters return their top results in rank + * order. The scorer applies the query's `k` cutoff; adapters are free + * to return fewer than k (with a shorter list). + */ + query(q: Query, state: BrainState): Promise<RankedDoc[]>; + + /** + * Persist the brain state to a filesystem path (for reproducibility + + * cross-task state sharing). Returns the path. Optional — adapters + * that don't support snapshotting can return an empty string. + */ + snapshot?(state: BrainState): Promise<string>; + + /** + * Per-page poison disposition: what did the adapter do with each + * poison item? Scorer compares to gold's `expected_behavior`. + * Adapters that don't have a poison path return an empty map. + */ + getPoisonDisposition?(state: BrainState): Record<string, PoisonDisposition>; +} + +// ─── Scorer helpers ───────────────────────────────────────────────── + +/** Standard top-K slice; helper since every scorer needs it. */ +export function topK(docs: RankedDoc[], k: number): RankedDoc[] { + return docs.slice(0, k); +} + +/** Precision@k: fraction of top-k that are in relevant set. */ +export function precisionAtK(docs: RankedDoc[], relevant: Set<string>, k: number): number { + const topDocs = topK(docs, k); + if (topDocs.length === 0) return 0; + let hits = 0; + for (const d of topDocs) if (relevant.has(d.page_id)) hits++; + return hits / topDocs.length; +} + +/** Recall@k: fraction of relevant found in top-k. */ +export function recallAtK(docs: RankedDoc[], relevant: Set<string>, k: number): number { + if (relevant.size === 0) return 0; + const topDocs = topK(docs, k); + let hits = 0; + for (const d of topDocs) if (relevant.has(d.page_id)) hits++; + return hits / relevant.size; +}