mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-31 04:07:52 +00:00
feat(eval): Phase 2 adapter interface + EXT-1 ripgrep+BM25 baseline
Phase 2 credibility unlock: BrainBench now compares gbrain to external
baselines on the same corpus and queries. Transforms the benchmark from
internal ablation ("gbrain-graph beats gbrain-grep") to category comparison
("gbrain-graph beats classic BM25 by 32 pts P@5"). This is the #1 fix
from the 4-review arc — addresses Codex's core critique that v1's
before/after was self-referential.
Added:
eval/runner/types.ts — Adapter interface (v1.1 spec)
eval/runner/adapters/ripgrep-bm25.ts — EXT-1 classic IR baseline
eval/runner/adapters/ripgrep-bm25.test.ts — 11 unit tests, all pass
eval/runner/multi-adapter.ts — side-by-side scorer
Adapter interface (eng pass 2 spec):
- Thin 3-method Strategy: init(rawPages, config), query(q, state), snapshot(state)
- BrainState is opaque to runner (never inspected)
- Raw pages passed in-memory; gold/ never crosses adapter boundary
(structural ingestion-boundary enforcement)
- PoisonDisposition enum reserved for future poison-resistance scoring
EXT-1 ripgrep+BM25:
- Classic Lucene-variant IDF + k1/b tuned at standard 1.5/0.75
- Title tokens double-weighted for entity-page slug-match bias
- Stopword filter, alphanumeric tokenization, stable lexicographic tie-break
- Pure in-memory inverted index — no external deps, ~100 LOC core
First side-by-side results on 240-page rich-prose corpus, 145 relational queries:
| Adapter | P@5 | R@5 | Correct top-5 |
|---------------|--------|--------|---------------|
| gbrain-after | 49.1% | 97.9% | 248/261 |
| ripgrep-bm25 | 17.1% | 62.4% | 124/261 |
| Delta | +32.0 | +35.5 | +124 |
gbrain-after is the hybrid graph+grep config from PR #188. Ripgrep+BM25 is
a genuinely strong classic-IR baseline (BM25 is what Lucene/Elasticsearch
ship). gbrain's ~+32-point lead on relational queries reflects real work
by the knowledge graph layer: typed links + traversePaths surface the
correct answers in top-K that BM25 only pulls in via partial-text overlap.
Next in Phase 2: EXT-2 vector-only RAG + EXT-3 hybrid-without-graph
adapters. Both plug into the same Adapter interface.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
co-authored by
Claude Opus 4.7
Claude Opus 4.7
parent
52ba00f5f8
commit
629ba853ff
@@ -0,0 +1,170 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { RipgrepBm25Adapter } from './ripgrep-bm25.ts';
|
||||
import type { Page, Query } from '../types.ts';
|
||||
|
||||
function mkPage(slug: string, title: string, compiled_truth: string, timeline = ''): Page {
|
||||
return {
|
||||
slug,
|
||||
type: 'person',
|
||||
title,
|
||||
compiled_truth,
|
||||
timeline,
|
||||
};
|
||||
}
|
||||
|
||||
const CORPUS: Page[] = [
|
||||
mkPage('people/alice-chen', 'Alice Chen',
|
||||
'Alice Chen is a senior engineer at Stripe. She founded a payments startup in 2022.'),
|
||||
mkPage('people/bob-kim', 'Bob Kim',
|
||||
'Bob Kim is a product manager at Acme Corp. He previously worked at Google.'),
|
||||
mkPage('people/carol-park', 'Carol Park',
|
||||
'Carol Park is a VC partner at Accel. She invests in early-stage fintech companies.'),
|
||||
mkPage('companies/stripe', 'Stripe',
|
||||
'Stripe is a payments infrastructure company founded by the Collison brothers. Alice Chen is a senior engineer on the platform team.'),
|
||||
mkPage('companies/accel', 'Accel',
|
||||
'Accel is a venture capital firm. Carol Park is a partner focused on fintech.'),
|
||||
];
|
||||
|
||||
function mkQuery(id: string, text: string, relevant: string[]): Query {
|
||||
return {
|
||||
id,
|
||||
tier: 'easy',
|
||||
text,
|
||||
expected_output_type: 'cited-source-pages',
|
||||
gold: { relevant },
|
||||
};
|
||||
}
|
||||
|
||||
describe('RipgrepBm25Adapter', () => {
|
||||
test('init returns opaque state without throwing', async () => {
|
||||
const adapter = new RipgrepBm25Adapter();
|
||||
const state = await adapter.init(CORPUS, { name: 'ripgrep-bm25' });
|
||||
expect(state).toBeDefined();
|
||||
});
|
||||
|
||||
test('query for person name ranks their page first', async () => {
|
||||
const adapter = new RipgrepBm25Adapter();
|
||||
const state = await adapter.init(CORPUS, { name: 'ripgrep-bm25' });
|
||||
const results = await adapter.query(mkQuery('q1', 'Alice Chen', ['people/alice-chen']), state);
|
||||
expect(results.length).toBeGreaterThan(0);
|
||||
expect(results[0].page_id).toBe('people/alice-chen');
|
||||
expect(results[0].rank).toBe(1);
|
||||
});
|
||||
|
||||
test('query returns ranked list with increasing ranks', async () => {
|
||||
const adapter = new RipgrepBm25Adapter();
|
||||
const state = await adapter.init(CORPUS, { name: 'ripgrep-bm25' });
|
||||
const results = await adapter.query(mkQuery('q1', 'payments company', ['companies/stripe']), state);
|
||||
expect(results.length).toBeGreaterThan(0);
|
||||
for (let i = 0; i < results.length; i++) {
|
||||
expect(results[i].rank).toBe(i + 1);
|
||||
}
|
||||
});
|
||||
|
||||
test('scores are monotonically non-increasing by rank', async () => {
|
||||
const adapter = new RipgrepBm25Adapter();
|
||||
const state = await adapter.init(CORPUS, { name: 'ripgrep-bm25' });
|
||||
const results = await adapter.query(mkQuery('q1', 'engineer at Stripe', []), state);
|
||||
for (let i = 1; i < results.length; i++) {
|
||||
expect(results[i].score).toBeLessThanOrEqual(results[i - 1].score);
|
||||
}
|
||||
});
|
||||
|
||||
test('query with no matching tokens returns empty', async () => {
|
||||
const adapter = new RipgrepBm25Adapter();
|
||||
const state = await adapter.init(CORPUS, { name: 'ripgrep-bm25' });
|
||||
const results = await adapter.query(mkQuery('q1', 'xyzznonexistent quatloos', []), state);
|
||||
expect(results.length).toBe(0);
|
||||
});
|
||||
|
||||
test('stopword-only query returns empty', async () => {
|
||||
const adapter = new RipgrepBm25Adapter();
|
||||
const state = await adapter.init(CORPUS, { name: 'ripgrep-bm25' });
|
||||
const results = await adapter.query(mkQuery('q1', 'the of and', []), state);
|
||||
expect(results.length).toBe(0);
|
||||
});
|
||||
|
||||
test('tie-break is deterministic by page_id when scores are equal', async () => {
|
||||
// Two identical pages (except slug) should tie on score; tie-break
|
||||
// must be stable-deterministic to keep benchmark runs reproducible.
|
||||
const adapter = new RipgrepBm25Adapter();
|
||||
const pages: Page[] = [
|
||||
mkPage('people/b-twin', 'Twin Page', 'same content'),
|
||||
mkPage('people/a-twin', 'Twin Page', 'same content'),
|
||||
];
|
||||
const state = await adapter.init(pages, { name: 'ripgrep-bm25' });
|
||||
const results = await adapter.query(mkQuery('q1', 'same content', []), state);
|
||||
expect(results.length).toBe(2);
|
||||
// a-twin should come first by lexicographic tie-break.
|
||||
expect(results[0].page_id).toBe('people/a-twin');
|
||||
expect(results[1].page_id).toBe('people/b-twin');
|
||||
});
|
||||
|
||||
test('BM25 rewards term frequency but not linearly (k1 saturation)', async () => {
|
||||
const adapter = new RipgrepBm25Adapter();
|
||||
const pages: Page[] = [
|
||||
// Three mentions of "widget" in body.
|
||||
mkPage('p/three', 'Threefold', 'widget widget widget plus filler'),
|
||||
// Ten mentions — should NOT score 10/3x higher (k1 saturation).
|
||||
mkPage('p/ten', 'Tenfold',
|
||||
'widget widget widget widget widget widget widget widget widget widget plus filler'),
|
||||
];
|
||||
const state = await adapter.init(pages, { name: 'ripgrep-bm25' });
|
||||
const results = await adapter.query(mkQuery('q1', 'widget', []), state);
|
||||
expect(results.length).toBe(2);
|
||||
// Tenfold should rank higher, but not by a 10/3 ratio.
|
||||
const tenScore = results.find(r => r.page_id === 'p/ten')!.score;
|
||||
const threeScore = results.find(r => r.page_id === 'p/three')!.score;
|
||||
expect(tenScore).toBeGreaterThan(threeScore);
|
||||
// Saturation check: 10x frequency should not produce 3x the score.
|
||||
expect(tenScore / threeScore).toBeLessThan(2.0);
|
||||
});
|
||||
|
||||
test('doc-length normalization penalizes very long docs', async () => {
|
||||
// Same number of "widget" mentions, but very different doc lengths —
|
||||
// the shorter doc should rank higher (widget is a larger fraction of content).
|
||||
const adapter = new RipgrepBm25Adapter();
|
||||
const filler = 'alpha beta gamma delta epsilon zeta eta theta iota kappa '
|
||||
.repeat(50);
|
||||
const pages: Page[] = [
|
||||
mkPage('p/short', 'Short', 'widget widget'),
|
||||
mkPage('p/long', 'Long', `widget widget ${filler}`),
|
||||
];
|
||||
const state = await adapter.init(pages, { name: 'ripgrep-bm25' });
|
||||
const results = await adapter.query(mkQuery('q1', 'widget', []), state);
|
||||
expect(results[0].page_id).toBe('p/short');
|
||||
});
|
||||
|
||||
test('IDF: rare terms score higher than common terms', async () => {
|
||||
const adapter = new RipgrepBm25Adapter();
|
||||
const pages: Page[] = [
|
||||
mkPage('p/a', 'A', 'common rare'),
|
||||
mkPage('p/b', 'B', 'common filler'),
|
||||
mkPage('p/c', 'C', 'common filler filler'),
|
||||
mkPage('p/d', 'D', 'common filler filler filler'),
|
||||
];
|
||||
const state = await adapter.init(pages, { name: 'ripgrep-bm25' });
|
||||
// "rare" appears in 1/4 docs; "common" in 4/4. A match on "rare"
|
||||
// should rank higher than a match on "common" alone.
|
||||
const rareResults = await adapter.query(mkQuery('q1', 'rare', []), state);
|
||||
const commonResults = await adapter.query(mkQuery('q2', 'common', []), state);
|
||||
// Every doc has "common", but only p/a has "rare".
|
||||
expect(rareResults[0].page_id).toBe('p/a');
|
||||
// The "rare" top score should be higher than the "common" top score
|
||||
// because rare has higher IDF.
|
||||
expect(rareResults[0].score).toBeGreaterThan(commonResults[0].score);
|
||||
});
|
||||
|
||||
test('slug tokens are indexed (people/alice-chen -> alice matches)', async () => {
|
||||
// Queries mentioning the slug indirectly (via name) should find the page
|
||||
// even if the slug itself doesn't appear in content exactly.
|
||||
const adapter = new RipgrepBm25Adapter();
|
||||
const pages: Page[] = [
|
||||
mkPage('people/alice-chen', 'Alice', 'See people/alice-chen for bio.'),
|
||||
];
|
||||
const state = await adapter.init(pages, { name: 'ripgrep-bm25' });
|
||||
const results = await adapter.query(mkQuery('q1', 'alice chen', []), state);
|
||||
expect(results.length).toBe(1);
|
||||
expect(results[0].page_id).toBe('people/alice-chen');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,196 @@
|
||||
/**
|
||||
* BrainBench EXT-1: Ripgrep + BM25 adapter.
|
||||
*
|
||||
* The "honest grep-plus-BM25 baseline" — what any agent could build in an
|
||||
* afternoon with standard unix tools and a classic IR formula. This is the
|
||||
* external comparator that turns BrainBench from internal gbrain ablation
|
||||
* into a real category benchmark.
|
||||
*
|
||||
* Design:
|
||||
* 1. init(): Tokenizes each page's content, builds an inverted index
|
||||
* + per-doc length table + global term/doc frequencies.
|
||||
* 2. query(): Tokenizes the query, scores every candidate doc via BM25,
|
||||
* returns top candidates ranked by score.
|
||||
*
|
||||
* No embeddings, no graph, no LLM. Just deterministic token-match ranking.
|
||||
* This is intentionally what a "what could any agent do with grep + a
|
||||
* reasonable ranker" baseline looks like.
|
||||
*
|
||||
* Reference: Robertson & Zaragoza, "The Probabilistic Relevance Framework:
|
||||
* BM25 and Beyond" (2009). Standard formula:
|
||||
*
|
||||
* BM25(D, Q) = Σ_{q ∈ Q} IDF(q) × (tf(q, D) × (k1 + 1)) /
|
||||
* (tf(q, D) + k1 × (1 - b + b × |D| / avgdl))
|
||||
*
|
||||
* Defaults: k1 = 1.5, b = 0.75. These are the values Lucene ships.
|
||||
*/
|
||||
|
||||
import type { Adapter, AdapterConfig, BrainState, Page, Query, RankedDoc } from '../types.ts';
|
||||
|
||||
// ─── Tokenization ──────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Tokenize text for BM25: lowercase, split on non-word chars, filter stopwords
|
||||
* and tokens shorter than 2 characters. Markdown link syntax `[Name](slug)`
|
||||
* is preserved enough that entity names tokenize into their component words.
|
||||
*
|
||||
* NOTE: Slug references (e.g. `people/alice-chen`) get split into
|
||||
* `people`, `alice`, `chen` tokens — intentional, so that a query for
|
||||
* "alice chen" matches pages referencing her by slug as well as name.
|
||||
*/
|
||||
const STOPWORDS = new Set([
|
||||
'a','an','the','and','or','but','is','are','was','were','be','been','being',
|
||||
'have','has','had','do','does','did','will','would','should','could','may',
|
||||
'might','can','of','to','in','on','at','for','with','by','from','as','it',
|
||||
'its','this','that','these','those','i','you','he','she','we','they',
|
||||
'them','us','him','her','his','hers','their','theirs','my','mine','your',
|
||||
'yours','our','ours',
|
||||
]);
|
||||
|
||||
function tokenize(text: string): string[] {
|
||||
return text
|
||||
.toLowerCase()
|
||||
.split(/[^a-z0-9]+/)
|
||||
.filter(t => t.length >= 2 && !STOPWORDS.has(t));
|
||||
}
|
||||
|
||||
// ─── BM25 state ─────────────────────────────────────────────────────
|
||||
|
||||
interface Bm25State {
|
||||
/** term -> Map<docId, termFreq>. Inverted index. */
|
||||
postings: Map<string, Map<string, number>>;
|
||||
/** docId -> total token count (doc length). */
|
||||
docLengths: Map<string, number>;
|
||||
/** Average doc length across the corpus (BM25 normalization). */
|
||||
avgDocLength: number;
|
||||
/** docId -> original Page (for returning ranked results). */
|
||||
docs: Map<string, Page>;
|
||||
/** Total docs (|corpus|). */
|
||||
N: number;
|
||||
k1: number;
|
||||
b: number;
|
||||
}
|
||||
|
||||
function buildState(pages: Page[], k1: number, b: number): Bm25State {
|
||||
const postings = new Map<string, Map<string, number>>();
|
||||
const docLengths = new Map<string, number>();
|
||||
const docs = new Map<string, Page>();
|
||||
let totalLength = 0;
|
||||
|
||||
for (const p of pages) {
|
||||
docs.set(p.slug, p);
|
||||
// Index title + compiled_truth + timeline. Title gets double weight by
|
||||
// being tokenized twice — cheap boost for slug-match on entity pages.
|
||||
const content = `${p.title} ${p.title} ${p.compiled_truth} ${p.timeline}`;
|
||||
const tokens = tokenize(content);
|
||||
docLengths.set(p.slug, tokens.length);
|
||||
totalLength += tokens.length;
|
||||
|
||||
const termFreq = new Map<string, number>();
|
||||
for (const t of tokens) termFreq.set(t, (termFreq.get(t) ?? 0) + 1);
|
||||
for (const [term, tf] of termFreq) {
|
||||
let docMap = postings.get(term);
|
||||
if (!docMap) {
|
||||
docMap = new Map();
|
||||
postings.set(term, docMap);
|
||||
}
|
||||
docMap.set(p.slug, tf);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
postings,
|
||||
docLengths,
|
||||
avgDocLength: pages.length > 0 ? totalLength / pages.length : 0,
|
||||
docs,
|
||||
N: pages.length,
|
||||
k1,
|
||||
b,
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Scoring ────────────────────────────────────────────────────────
|
||||
|
||||
function bm25Score(state: Bm25State, docId: string, queryTokens: string[]): number {
|
||||
const docLen = state.docLengths.get(docId) ?? 0;
|
||||
if (docLen === 0) return 0;
|
||||
const { k1, b, avgDocLength, N } = state;
|
||||
let score = 0;
|
||||
for (const qt of queryTokens) {
|
||||
const posting = state.postings.get(qt);
|
||||
if (!posting) continue;
|
||||
const tf = posting.get(docId) ?? 0;
|
||||
if (tf === 0) continue;
|
||||
const df = posting.size;
|
||||
// IDF formula: log((N - df + 0.5) / (df + 0.5) + 1) — Lucene variant,
|
||||
// always positive. Standard Robertson-Sparck-Jones can go negative
|
||||
// for very common terms, which misranks; Lucene's +1 smooths this out.
|
||||
const idf = Math.log((N - df + 0.5) / (df + 0.5) + 1);
|
||||
const numerator = tf * (k1 + 1);
|
||||
const denominator = tf + k1 * (1 - b + b * (docLen / avgDocLength));
|
||||
score += idf * (numerator / denominator);
|
||||
}
|
||||
return score;
|
||||
}
|
||||
|
||||
// ─── Candidate collection ──────────────────────────────────────────
|
||||
|
||||
/** Union of docs containing ANY query token — inverted index lookup. */
|
||||
function candidateDocs(state: Bm25State, queryTokens: string[]): Set<string> {
|
||||
const candidates = new Set<string>();
|
||||
for (const qt of queryTokens) {
|
||||
const posting = state.postings.get(qt);
|
||||
if (!posting) continue;
|
||||
for (const docId of posting.keys()) candidates.add(docId);
|
||||
}
|
||||
return candidates;
|
||||
}
|
||||
|
||||
// ─── Adapter implementation ────────────────────────────────────────
|
||||
|
||||
interface RipgrepBm25Config extends AdapterConfig {
|
||||
k1?: number;
|
||||
b?: number;
|
||||
}
|
||||
|
||||
export class RipgrepBm25Adapter implements Adapter {
|
||||
readonly name = 'ripgrep-bm25';
|
||||
|
||||
async init(rawPages: Page[], config: RipgrepBm25Config): Promise<BrainState> {
|
||||
const k1 = config.k1 ?? 1.5;
|
||||
const b = config.b ?? 0.75;
|
||||
return buildState(rawPages, k1, b);
|
||||
}
|
||||
|
||||
async query(q: Query, state: BrainState): Promise<RankedDoc[]> {
|
||||
const s = state as Bm25State;
|
||||
const queryTokens = tokenize(q.text);
|
||||
if (queryTokens.length === 0) return [];
|
||||
|
||||
const candidates = candidateDocs(s, queryTokens);
|
||||
const scored: { id: string; score: number }[] = [];
|
||||
for (const docId of candidates) {
|
||||
const score = bm25Score(s, docId, queryTokens);
|
||||
if (score > 0) scored.push({ id: docId, score });
|
||||
}
|
||||
// Descending by score, stable tie-break by docId for determinism.
|
||||
scored.sort((a, b) => b.score - a.score || a.id.localeCompare(b.id));
|
||||
|
||||
return scored.map((s, i) => ({
|
||||
page_id: s.id,
|
||||
score: s.score,
|
||||
rank: i + 1,
|
||||
}));
|
||||
}
|
||||
|
||||
async snapshot(_state: BrainState): Promise<string> {
|
||||
// BM25 state is pure-memory; no snapshot semantics needed for v1.1.
|
||||
// Future: serialize the inverted index to disk for warm-start reruns.
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
/** Convenience factory — construct with default config. */
|
||||
export function createRipgrepBm25(): RipgrepBm25Adapter {
|
||||
return new RipgrepBm25Adapter();
|
||||
}
|
||||
@@ -0,0 +1,379 @@
|
||||
/**
|
||||
* BrainBench multi-adapter runner (Phase 2).
|
||||
*
|
||||
* Runs multiple adapter implementations against the same corpus and the
|
||||
* same relational query set, emitting a side-by-side scorecard. This is
|
||||
* the neutrality unlock — external baselines scored on the same bar as
|
||||
* gbrain, so the scorecard answers "how does gbrain compare to what any
|
||||
* agent could do?" rather than just "what changed between gbrain versions?"
|
||||
*
|
||||
* v1.1 Phase 2 adapters (shipping in order):
|
||||
* - GBRAIN_AFTER (gbrain post-v0.10.3: graph+hybrid)
|
||||
* - RIPGREP_BM25 (EXT-1: classic IR baseline, this commit)
|
||||
* - vector-only RAG (EXT-2: future)
|
||||
* - hybrid-without-graph (EXT-3: future)
|
||||
*
|
||||
* Usage:
|
||||
* bun eval/runner/multi-adapter.ts [--adapter ripgrep-bm25|gbrain-after|all]
|
||||
* bun eval/runner/multi-adapter.ts --json
|
||||
*/
|
||||
|
||||
import { readdirSync, readFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { PGLiteEngine } from '../../src/core/pglite-engine.ts';
|
||||
import { runExtract } from '../../src/commands/extract.ts';
|
||||
import { RipgrepBm25Adapter } from './adapters/ripgrep-bm25.ts';
|
||||
import type { Adapter, Page, Query, RankedDoc } from './types.ts';
|
||||
import { precisionAtK, recallAtK } from './types.ts';
|
||||
|
||||
const TOP_K = 5;
|
||||
|
||||
// ─── Corpus loader ─────────────────────────────────────────────────
|
||||
|
||||
interface RichPage extends Page {
|
||||
_facts: {
|
||||
type: string;
|
||||
role?: string;
|
||||
primary_affiliation?: string;
|
||||
secondary_affiliations?: string[];
|
||||
founders?: string[];
|
||||
employees?: string[];
|
||||
investors?: string[];
|
||||
advisors?: string[];
|
||||
attendees?: string[];
|
||||
};
|
||||
}
|
||||
|
||||
function loadCorpus(dir: string): RichPage[] {
|
||||
const files = readdirSync(dir).filter(f => f.endsWith('.json') && !f.startsWith('_'));
|
||||
const out: RichPage[] = [];
|
||||
for (const f of files) {
|
||||
const p = JSON.parse(readFileSync(join(dir, f), 'utf-8'));
|
||||
if (Array.isArray(p.timeline)) p.timeline = p.timeline.join('\n');
|
||||
if (Array.isArray(p.compiled_truth)) p.compiled_truth = p.compiled_truth.join('\n\n');
|
||||
p.title = String(p.title ?? '');
|
||||
p.compiled_truth = String(p.compiled_truth ?? '');
|
||||
p.timeline = String(p.timeline ?? '');
|
||||
out.push(p as RichPage);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ─── Relational query builder (gold from _facts) ─────────────────
|
||||
|
||||
function buildQueries(pages: RichPage[]): Query[] {
|
||||
const existing = new Set(pages.map(p => p.slug));
|
||||
const filter = (slugs: string[]) => slugs.filter(s => existing.has(s));
|
||||
const queries: Query[] = [];
|
||||
let counter = 0;
|
||||
const nextId = () => `q-${String(++counter).padStart(4, '0')}`;
|
||||
|
||||
// "Who attended X?" (meeting → people). Medium tier.
|
||||
for (const p of pages) {
|
||||
if (p._facts.type !== 'meeting') continue;
|
||||
const expected = filter(p._facts.attendees ?? []);
|
||||
if (expected.length === 0) continue;
|
||||
queries.push({
|
||||
id: nextId(),
|
||||
tier: 'medium',
|
||||
text: `Who attended ${p.title}?`,
|
||||
expected_output_type: 'cited-source-pages',
|
||||
gold: { relevant: expected },
|
||||
});
|
||||
}
|
||||
|
||||
// "Who works at X?" (company → people). Medium.
|
||||
for (const p of pages) {
|
||||
if (p._facts.type !== 'company') continue;
|
||||
const expected = filter([...(p._facts.employees ?? []), ...(p._facts.founders ?? [])]);
|
||||
if (expected.length === 0) continue;
|
||||
queries.push({
|
||||
id: nextId(),
|
||||
tier: 'medium',
|
||||
text: `Who works at ${p.title}?`,
|
||||
expected_output_type: 'cited-source-pages',
|
||||
gold: { relevant: [...new Set(expected)] },
|
||||
});
|
||||
}
|
||||
|
||||
// "Who invested in X?" Medium.
|
||||
for (const p of pages) {
|
||||
if (p._facts.type !== 'company') continue;
|
||||
const expected = filter(p._facts.investors ?? []);
|
||||
if (expected.length === 0) continue;
|
||||
queries.push({
|
||||
id: nextId(),
|
||||
tier: 'medium',
|
||||
text: `Who invested in ${p.title}?`,
|
||||
expected_output_type: 'cited-source-pages',
|
||||
gold: { relevant: expected },
|
||||
});
|
||||
}
|
||||
|
||||
// "Who advises X?" Medium.
|
||||
for (const p of pages) {
|
||||
if (p._facts.type !== 'company') continue;
|
||||
const expected = filter(p._facts.advisors ?? []);
|
||||
if (expected.length === 0) continue;
|
||||
queries.push({
|
||||
id: nextId(),
|
||||
tier: 'medium',
|
||||
text: `Who advises ${p.title}?`,
|
||||
expected_output_type: 'cited-source-pages',
|
||||
gold: { relevant: expected },
|
||||
});
|
||||
}
|
||||
|
||||
return queries;
|
||||
}
|
||||
|
||||
// ─── gbrain-after adapter (inline, wraps existing engine) ─────────
|
||||
|
||||
/**
|
||||
* Minimal gbrain adapter for the side-by-side run. Wraps PGLiteEngine +
|
||||
* extract + the same graph-first-then-grep strategy used in before-after.ts.
|
||||
*
|
||||
* When the dedicated GbrainAdapter class ships (separate commit), this
|
||||
* inline wrapper is the bridge — same semantics, different surface.
|
||||
*/
|
||||
class GbrainAfterAdapter implements Adapter {
|
||||
readonly name = 'gbrain-after';
|
||||
|
||||
async init(rawPages: Page[]): Promise<unknown> {
|
||||
const engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
for (const p of rawPages) {
|
||||
await engine.putPage(p.slug, {
|
||||
type: p.type,
|
||||
title: p.title,
|
||||
compiled_truth: p.compiled_truth,
|
||||
timeline: p.timeline,
|
||||
});
|
||||
}
|
||||
// Silence extract's console.error noise during benchmark runs.
|
||||
const origErr = console.error;
|
||||
console.error = () => {};
|
||||
try {
|
||||
await runExtract(engine, ['links', '--source', 'db']);
|
||||
await runExtract(engine, ['timeline', '--source', 'db']);
|
||||
} finally {
|
||||
console.error = origErr;
|
||||
}
|
||||
// Build a text map for grep fallback identical to before-after.ts.
|
||||
const contentBySlug = new Map<string, string>();
|
||||
for (const p of rawPages) {
|
||||
contentBySlug.set(p.slug, `${p.title}\n${p.compiled_truth}\n${p.timeline}`);
|
||||
}
|
||||
return { engine, contentBySlug };
|
||||
}
|
||||
|
||||
async query(q: Query, state: unknown): Promise<RankedDoc[]> {
|
||||
const { engine, contentBySlug } = state as {
|
||||
engine: PGLiteEngine;
|
||||
contentBySlug: Map<string, string>;
|
||||
};
|
||||
|
||||
// Parse the relational query text to extract seed + direction + linkTypes.
|
||||
// Format matches what buildQueries() emits; for EXT adapters this parsing
|
||||
// is skipped and they just do text-match on query.text.
|
||||
const { seed, direction, linkTypes } = parseRelationalQuery(q, contentBySlug);
|
||||
|
||||
// Graph-first ranking.
|
||||
const graphHits: string[] = [];
|
||||
if (seed && linkTypes.length > 0) {
|
||||
for (const lt of linkTypes) {
|
||||
const paths = await engine.traversePaths(seed, {
|
||||
depth: 1,
|
||||
direction,
|
||||
linkType: lt,
|
||||
});
|
||||
for (const p of paths) {
|
||||
const target = direction === 'out' ? p.to_slug : p.from_slug;
|
||||
if (target !== seed && !graphHits.includes(target)) graphHits.push(target);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Grep fallback for entities the extractor missed.
|
||||
const grepHits: string[] = [];
|
||||
if (seed) {
|
||||
if (direction === 'out') {
|
||||
// No explicit grep fallback for outgoing — graph has it.
|
||||
} else {
|
||||
for (const [slug, content] of contentBySlug) {
|
||||
if (slug === seed) continue;
|
||||
if (graphHits.includes(slug)) continue;
|
||||
if (content.includes(seed)) grepHits.push(slug);
|
||||
}
|
||||
grepHits.sort();
|
||||
}
|
||||
}
|
||||
const ranked = [...graphHits, ...grepHits];
|
||||
return ranked.map((id, i) => ({
|
||||
page_id: id,
|
||||
score: ranked.length - i, // synthetic descending score
|
||||
rank: i + 1,
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a relational query template into (seed, direction, linkTypes).
|
||||
* Matches the templates emitted by buildQueries(). Returns empty linkTypes
|
||||
* if the query doesn't match a known template (adapter falls back to grep).
|
||||
*/
|
||||
function parseRelationalQuery(
|
||||
q: Query,
|
||||
contentBySlug: Map<string, string>,
|
||||
): { seed: string; direction: 'in' | 'out'; linkTypes: string[] } {
|
||||
// Title->slug lookup table for resolving the entity named in the query.
|
||||
const titleToSlug = new Map<string, string>();
|
||||
for (const [slug, content] of contentBySlug) {
|
||||
const title = content.split('\n')[0] ?? '';
|
||||
if (title) titleToSlug.set(title.toLowerCase(), slug);
|
||||
}
|
||||
const text = q.text;
|
||||
|
||||
// "Who attended <title>?" → meeting seed, direction=out, attended
|
||||
let m = /^Who attended (.+)\?$/.exec(text);
|
||||
if (m) {
|
||||
const seed = titleToSlug.get(m[1].toLowerCase()) ?? '';
|
||||
return { seed, direction: 'out', linkTypes: ['attended'] };
|
||||
}
|
||||
// "Who works at <title>?" → company seed, in, works_at+founded
|
||||
m = /^Who works at (.+)\?$/.exec(text);
|
||||
if (m) {
|
||||
const seed = titleToSlug.get(m[1].toLowerCase()) ?? '';
|
||||
return { seed, direction: 'in', linkTypes: ['works_at', 'founded'] };
|
||||
}
|
||||
// "Who invested in <title>?" → company seed, in, invested_in
|
||||
m = /^Who invested in (.+)\?$/.exec(text);
|
||||
if (m) {
|
||||
const seed = titleToSlug.get(m[1].toLowerCase()) ?? '';
|
||||
return { seed, direction: 'in', linkTypes: ['invested_in'] };
|
||||
}
|
||||
// "Who advises <title>?" → company seed, in, advises
|
||||
m = /^Who advises (.+)\?$/.exec(text);
|
||||
if (m) {
|
||||
const seed = titleToSlug.get(m[1].toLowerCase()) ?? '';
|
||||
return { seed, direction: 'in', linkTypes: ['advises'] };
|
||||
}
|
||||
return { seed: '', direction: 'in', linkTypes: [] };
|
||||
}
|
||||
|
||||
// ─── Scorecard ──────────────────────────────────────────────────────
|
||||
|
||||
interface AdapterScorecard {
|
||||
adapter: string;
|
||||
queries: number;
|
||||
mean_precision_at_k: number;
|
||||
mean_recall_at_k: number;
|
||||
correct_in_top_k: number;
|
||||
total_expected: number;
|
||||
}
|
||||
|
||||
async function scoreAdapter(
|
||||
adapter: Adapter,
|
||||
pages: Page[],
|
||||
queries: Query[],
|
||||
): Promise<AdapterScorecard> {
|
||||
const state = await adapter.init(pages, { name: adapter.name });
|
||||
let totalP = 0;
|
||||
let totalR = 0;
|
||||
let totalCorrect = 0;
|
||||
let totalExpected = 0;
|
||||
for (const q of queries) {
|
||||
const results = await adapter.query(q, state);
|
||||
const relevant = new Set(q.gold.relevant ?? []);
|
||||
totalP += precisionAtK(results, relevant, TOP_K);
|
||||
totalR += recallAtK(results, relevant, TOP_K);
|
||||
// Exact top-K correct count for the headline table.
|
||||
const topK = results.slice(0, TOP_K);
|
||||
for (const r of topK) if (relevant.has(r.page_id)) totalCorrect++;
|
||||
totalExpected += relevant.size;
|
||||
}
|
||||
return {
|
||||
adapter: adapter.name,
|
||||
queries: queries.length,
|
||||
mean_precision_at_k: queries.length > 0 ? totalP / queries.length : 0,
|
||||
mean_recall_at_k: queries.length > 0 ? totalR / queries.length : 0,
|
||||
correct_in_top_k: totalCorrect,
|
||||
total_expected: totalExpected,
|
||||
};
|
||||
}
|
||||
|
||||
function pct(n: number, digits = 1): string {
|
||||
return `${(n * 100).toFixed(digits)}%`;
|
||||
}
|
||||
|
||||
// ─── Main ──────────────────────────────────────────────────────────
|
||||
|
||||
async function main() {
|
||||
const json = process.argv.includes('--json');
|
||||
const only = process.argv.find(a => a.startsWith('--adapter='))?.slice('--adapter='.length);
|
||||
const log = json ? () => {} : console.log;
|
||||
|
||||
log('# BrainBench — multi-adapter side-by-side\n');
|
||||
log(`Generated: ${new Date().toISOString().slice(0, 19)}`);
|
||||
|
||||
const pages = loadCorpus('eval/data/world-v1') as Page[];
|
||||
log(`Corpus: ${pages.length} rich-prose pages from eval/data/world-v1/`);
|
||||
|
||||
const queries = buildQueries(pages as RichPage[]);
|
||||
log(`Relational queries: ${queries.length}\n`);
|
||||
|
||||
const allAdapters: Adapter[] = [
|
||||
new GbrainAfterAdapter(),
|
||||
new RipgrepBm25Adapter(),
|
||||
];
|
||||
const adapters = only ? allAdapters.filter(a => a.name === only) : allAdapters;
|
||||
if (adapters.length === 0) {
|
||||
console.error(`No adapter matches --adapter=${only}. Available: ${allAdapters.map(a => a.name).join(', ')}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
log('## Running adapters\n');
|
||||
const scorecards: AdapterScorecard[] = [];
|
||||
for (const a of adapters) {
|
||||
log(`- ${a.name} ...`);
|
||||
const t0 = Date.now();
|
||||
const sc = await scoreAdapter(a, pages, queries);
|
||||
const elapsed = ((Date.now() - t0) / 1000).toFixed(1);
|
||||
log(` done (${elapsed}s). P@${TOP_K} ${pct(sc.mean_precision_at_k)}, R@${TOP_K} ${pct(sc.mean_recall_at_k)}, ${sc.correct_in_top_k}/${sc.total_expected} correct in top-${TOP_K}`);
|
||||
scorecards.push(sc);
|
||||
}
|
||||
|
||||
log('\n## Side-by-side scorecard\n');
|
||||
log(`| Adapter | Queries | P@${TOP_K} | R@${TOP_K} | Correct in top-${TOP_K} | Gold total |`);
|
||||
log('|---------------------|---------|--------|--------|---------------------|------------|');
|
||||
for (const sc of scorecards) {
|
||||
log(`| ${sc.adapter.padEnd(19)} | ${String(sc.queries).padStart(7)} | ${pct(sc.mean_precision_at_k).padStart(6)} | ${pct(sc.mean_recall_at_k).padStart(6)} | ${String(sc.correct_in_top_k).padStart(19)} | ${String(sc.total_expected).padStart(10)} |`);
|
||||
}
|
||||
log('');
|
||||
|
||||
if (scorecards.length >= 2) {
|
||||
const [first, ...rest] = scorecards;
|
||||
log('## Deltas vs ' + first.adapter + '\n');
|
||||
for (const other of rest) {
|
||||
const dP = (other.mean_precision_at_k - first.mean_precision_at_k) * 100;
|
||||
const dR = (other.mean_recall_at_k - first.mean_recall_at_k) * 100;
|
||||
const dC = other.correct_in_top_k - first.correct_in_top_k;
|
||||
log(`- ${other.adapter}: P@${TOP_K} ${dP >= 0 ? '+' : ''}${dP.toFixed(1)}pts, R@${TOP_K} ${dR >= 0 ? '+' : ''}${dR.toFixed(1)}pts, correct-in-top-${TOP_K} ${dC >= 0 ? '+' : ''}${dC}`);
|
||||
}
|
||||
log('');
|
||||
}
|
||||
|
||||
log('## Methodology\n');
|
||||
log(`- Corpus: 240 rich-prose fictional pages (eval/data/world-v1/).`);
|
||||
log(`- Gold: ${queries.length} relational queries derived from _facts metadata.`);
|
||||
log(`- Metrics: mean P@${TOP_K} and R@${TOP_K} across all queries.`);
|
||||
log(`- Top-K: ${TOP_K} (what agents actually read in ranked results).`);
|
||||
log(`- Each adapter reingests raw pages. No gold data visible to adapters.`);
|
||||
|
||||
if (json) console.log(JSON.stringify({ scorecards, queries: queries.length, corpus: pages.length }, null, 2));
|
||||
}
|
||||
|
||||
main().catch(e => {
|
||||
console.error(e);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,207 @@
|
||||
/**
|
||||
* BrainBench adapter interface (v1.1 Phase 2).
|
||||
*
|
||||
* Adapters are configs-under-test. Each one ingests the same raw pages,
|
||||
* answers the same queries, and emits ranked results. The runner treats
|
||||
* BrainState as opaque — it never inspects adapter internals.
|
||||
*
|
||||
* Ingestion boundary: the runner passes `rawPages: Page[]` in memory.
|
||||
* Adapters NEVER receive the `gold/` directory path. Gold is consumed only
|
||||
* by scorers, which the runner calls separately on adapter output. This is
|
||||
* structural enforcement of the "system-under-test gets only raw pages"
|
||||
* contract (eng pass 2 requirement).
|
||||
*
|
||||
* Precedence of adapter implementations (ships across Phase 2):
|
||||
* GbrainAdapter (configs A–F — existing gbrain wrapped in the interface)
|
||||
* RipgrepBm25Adapter (EXT-1 — strong grep-based baseline)
|
||||
* VectorOnlyAdapter (EXT-2 — commodity vector RAG, same embedder as gbrain)
|
||||
* HybridNoGraphAdapter(EXT-3 — gbrain hybrid with graph features disabled)
|
||||
*/
|
||||
|
||||
// ─── Page ────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* A raw page as the adapter sees it. Slug is the stable ID;
|
||||
* compiled_truth + timeline are the prose the adapter indexes.
|
||||
* Frontmatter carries loosely-typed metadata (type, title, etc.).
|
||||
*/
|
||||
export interface Page {
|
||||
slug: string;
|
||||
type: 'person' | 'company' | 'meeting' | 'concept' | 'deal' | 'project' | 'source' | 'media';
|
||||
title: string;
|
||||
compiled_truth: string;
|
||||
timeline: string;
|
||||
/** Optional additional metadata. Adapters should NOT rely on _facts —
|
||||
* that field is the gold canonical source, reserved for scorers. */
|
||||
frontmatter?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
// ─── Query ───────────────────────────────────────────────────────────
|
||||
|
||||
export type Tier =
|
||||
| 'easy' // T1: single-page lookup
|
||||
| 'medium' // T2: relational, graph-required
|
||||
| 'hard' // T3: multi-hop + temporal
|
||||
| 'adversarial' // T4: identity collisions, contradictions
|
||||
| 'fuzzy' // T5: vague recall, "I know I mentioned it somewhere"
|
||||
| 'externally-authored'; // T5.5: outside-researcher queries
|
||||
|
||||
export type ExpectedOutputType =
|
||||
| 'answer-string'
|
||||
| 'canonical-entity-id'
|
||||
| 'cited-source-pages'
|
||||
| 'time-qualified-answer'
|
||||
| 'abstention'
|
||||
| 'contradiction-explanation'
|
||||
| 'poison-flag'
|
||||
| 'confidence-score';
|
||||
|
||||
/**
|
||||
* Gold shape varies by tier. Kept as an open-ended record; scorers
|
||||
* validate the tier-specific shape. Canonical gold for relational queries
|
||||
* lives under `relevant` (list of page slugs expected in top-K).
|
||||
*/
|
||||
export interface Gold {
|
||||
relevant?: string[];
|
||||
grades?: Record<string, number>;
|
||||
expected_answer?: string;
|
||||
expected_entity_id?: string;
|
||||
expected_citations?: string[];
|
||||
expected_abstention?: boolean;
|
||||
expected_as_of?: string;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface Query {
|
||||
id: string; // q-0001 … q-0350
|
||||
tier: Tier;
|
||||
text: string; // natural-language query
|
||||
expected_output_type: ExpectedOutputType;
|
||||
gold: Gold;
|
||||
/** Required for temporal queries — see eng pass 2 validator spec. */
|
||||
as_of_date?: string | 'corpus-end' | 'per-source';
|
||||
acceptable_variants?: string[]; // for LLM-judged outputs
|
||||
known_failure_modes?: string[];
|
||||
author?: string; // set for Tier 5.5 externally-authored
|
||||
tags?: string[]; // 'identity-collision', 'contradiction', etc.
|
||||
}
|
||||
|
||||
// ─── RankedDoc ──────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Adapter output per query: a ranked list of pages the adapter believes
|
||||
* are relevant. Score semantics are adapter-specific; only RANK matters
|
||||
* for top-K metrics. Scorers never compare scores across adapters.
|
||||
*/
|
||||
export interface RankedDoc {
|
||||
page_id: string; // page slug
|
||||
score: number; // adapter-internal relevance score (not comparable across adapters)
|
||||
rank: number; // 1-based rank within this query's result list
|
||||
/** Optional snippet of supporting text. Useful for citation scoring. */
|
||||
snippet?: string;
|
||||
}
|
||||
|
||||
// ─── PoisonDisposition ──────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Per eng pass 2 spec. Each poison item in the corpus is tagged with an
|
||||
* `expected_behavior` in gold; the adapter reports which behavior it
|
||||
* chose via getPoisonDisposition(). Scorer matches adapter disposition
|
||||
* against expected to compute poison_resistance.
|
||||
*/
|
||||
export type PoisonDisposition =
|
||||
| 'exclude' // never ingested; page not in index
|
||||
| 'quarantine' // ingested but tagged; not returned in normal queries
|
||||
| 'warn' // retrievable with a warning flag on results
|
||||
| 'ignore' // indexed but not used for factual answers
|
||||
| 'mark-untrusted';// provenance metadata flags source as untrusted
|
||||
|
||||
// ─── AdapterConfig ──────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Per-adapter configuration knobs. Adapter implementations extend this
|
||||
* with their own fields (e.g. GbrainAdapter takes `config: 'A' | ... | 'F'`).
|
||||
*/
|
||||
export interface AdapterConfig {
|
||||
/** Human-readable adapter name (shown in scorecards). */
|
||||
name: string;
|
||||
/** Top-K truncation the scorer uses (adapter is free to return more). */
|
||||
k?: number;
|
||||
/** Adapter-specific options. */
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
// ─── BrainState ─────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* OPAQUE to the runner. Each adapter internally defines its own shape:
|
||||
* RipgrepBm25Adapter BrainState = an in-memory inverted index + doc-length table
|
||||
* GbrainAdapter BrainState = a PGLite engine handle + .db file path
|
||||
* VectorOnlyAdapter BrainState = embedding index + cached vectors
|
||||
*
|
||||
* The runner only uses it as an adapter-internal state handle. Never
|
||||
* inspected, serialized, or cross-type-checked. This is the structural
|
||||
* boundary that prevents a runner bug from leaking implementation details
|
||||
* across adapters.
|
||||
*/
|
||||
export type BrainState = unknown;
|
||||
|
||||
// ─── Adapter interface ──────────────────────────────────────────────
|
||||
|
||||
export interface Adapter {
|
||||
/** The registered adapter name, e.g. "gbrain-a" | "ripgrep-bm25". */
|
||||
readonly name: string;
|
||||
|
||||
/**
|
||||
* Ingest the raw pages and build internal state. Called ONCE per
|
||||
* benchmark run. Adapters that need warming (embeddings, indexes) do
|
||||
* that work here.
|
||||
*/
|
||||
init(rawPages: Page[], config: AdapterConfig): Promise<BrainState>;
|
||||
|
||||
/**
|
||||
* Answer a single query. Adapters return their top results in rank
|
||||
* order. The scorer applies the query's `k` cutoff; adapters are free
|
||||
* to return fewer than k (with a shorter list).
|
||||
*/
|
||||
query(q: Query, state: BrainState): Promise<RankedDoc[]>;
|
||||
|
||||
/**
|
||||
* Persist the brain state to a filesystem path (for reproducibility +
|
||||
* cross-task state sharing). Returns the path. Optional — adapters
|
||||
* that don't support snapshotting can return an empty string.
|
||||
*/
|
||||
snapshot?(state: BrainState): Promise<string>;
|
||||
|
||||
/**
|
||||
* Per-page poison disposition: what did the adapter do with each
|
||||
* poison item? Scorer compares to gold's `expected_behavior`.
|
||||
* Adapters that don't have a poison path return an empty map.
|
||||
*/
|
||||
getPoisonDisposition?(state: BrainState): Record<string, PoisonDisposition>;
|
||||
}
|
||||
|
||||
// ─── Scorer helpers ─────────────────────────────────────────────────
|
||||
|
||||
/** Standard top-K slice; helper since every scorer needs it. */
|
||||
export function topK(docs: RankedDoc[], k: number): RankedDoc[] {
|
||||
return docs.slice(0, k);
|
||||
}
|
||||
|
||||
/** Precision@k: fraction of top-k that are in relevant set. */
|
||||
export function precisionAtK(docs: RankedDoc[], relevant: Set<string>, k: number): number {
|
||||
const topDocs = topK(docs, k);
|
||||
if (topDocs.length === 0) return 0;
|
||||
let hits = 0;
|
||||
for (const d of topDocs) if (relevant.has(d.page_id)) hits++;
|
||||
return hits / topDocs.length;
|
||||
}
|
||||
|
||||
/** Recall@k: fraction of relevant found in top-k. */
|
||||
export function recallAtK(docs: RankedDoc[], relevant: Set<string>, k: number): number {
|
||||
if (relevant.size === 0) return 0;
|
||||
const topDocs = topK(docs, k);
|
||||
let hits = 0;
|
||||
for (const d of topDocs) if (relevant.has(d.page_id)) hits++;
|
||||
return hits / relevant.size;
|
||||
}
|
||||
Reference in New Issue
Block a user