admin/gbrain
1
0
Fork 0
mirror of https://github.com/garrytan/gbrain.git synced 2026-07-31 04:07:52 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
a4cdb41b0771efde56ef9bd5e42085a6dd45ffde
gbrain/test/eval/cat11.test.ts
T
Garry TanandClaude Opus 4.6 839258dc42 feat(eval): Day 6 — adversarial-injections + Cat 6 prose-scale + Cat 11 multi-modal 
Three modules that together cover BrainBench v1 Cat 6 (prose-scale
extraction fidelity) and Cat 11 (multi-modal ingest fidelity).

**eval/runner/adversarial-injections.ts** — 6 deterministic content
transforms shared by Cat 10 (adversarial.ts, 22 hand-crafted cases) and
Cat 6 (prose-scale variants). Each injection produces a modified content
string + a structured GoldDelta describing what the extractor MUST and
MUST NOT produce. Kinds:
  - code_fence_leak — fake [X](people/fake) inside ``` fence, must NOT extract
  - inline_code_slug — `people/fake` in backticks, must NOT extract
  - substring_collision — "SamAI" near real `people/sam`, exactly one link
  - ambiguous_role — "works with" vs "works at", downgrade type to mentions
  - prose_only_mention — strip markdown link syntax, bare name → mentions only
  - multi_entity_sentence — pack 4+ entities into one clause, extract all

Mulberry32 PRNG keeps variant generation deterministic under fixed seed.
Codex flagged the original plan's wording ("extract injection engine from
adversarial.ts") as overstated — adversarial.ts is a static case list,
not a reusable engine. This module is NEW code.

**eval/runner/cat6-prose-scale.ts** — Runner. Loads world-v1, applies all
6 injection kinds to sampled base pages (default 50 variants per kind ×
6 kinds = 300 variants), runs extractPageLinks on each, compares to gold
delta. Emits per-kind + overall metrics (precision, recall, F1,
code_fence_leak_rate, substring_fp_rate, pages_with_links_coverage,
mean_links_per_page). **v1 verdict is always "baseline_only"** — no
gating threshold per codex fix #9 (current extractor residuals make
>0.80 unreachable; v1 records a baseline, regression guard triggers on
drop below it).

**eval/runner/cat11-multimodal.ts** — PDF + HTML + audio runners.
Fixtures load from eval/data/multimodal/<modality>/fixtures.json
manifests; each modality skips gracefully when manifest missing or
(audio) when neither GROQ_API_KEY nor OPENAI_API_KEY is set. Metrics:
  - PDF: char-level similarity via Levenshtein + optional entity_recall
  - HTML: word-recall over normalized tokens (multiset semantics)
  - Audio: WER (word error rate) via Levenshtein on word sequences
Fixtures are NOT committed; a future eval:fetch-multimodal script will
download them hash-verified from public sources (arXiv CC-licensed
papers, Wikipedia CC-BY-SA, Common Voice CC0).

Injectable audio transcriber (`opts.transcribe`) means tests don't need
GROQ/OpenAI keys — stubbed transcriptions exercise the WER math path
directly.

**Tests (60 new):** adversarial-injections (19) — per-kind assertions +
dispatcher coverage + slug regex conformance; cat6 (12) — variant
determinism, scoreVariant shape, aggregate per-kind + overall metrics,
corpus resolver slug rules; cat11 (29) — charSimilarity / wordRecall /
wer math, htmlToText strips scripts + decodes entities, HTML modality
with real fixtures, audio modality gracefully skips without key + uses
stub transcriber correctly.

All 60 tests pass in 48ms + 41ms.
Total eval suite now: 227 pass, 0 fail.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-04-20 21:59:30 +08:00

317 lines
11 KiB
TypeScript
Raw Blame History

/**
* cat11-multimodal.ts tests — Day 6 of BrainBench v1 Complete.
*
* Covers:
* - charSimilarity math: identical / empty / partial
* - wordRecall: multiset semantics, empty canonical
* - wer (word error rate): perfect / half-off / empty ref
* - htmlToText: strips script/style/tags, decodes common entities
* - runHtmlModality with a real fixture (tmpdir, hand-rolled HTML)
* - runAudioModality skips gracefully without API keys + manifest
* - runPdfModality skips gracefully without manifest
* - runAudioModality uses injected transcriber stub
*/
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
import { mkdirSync, writeFileSync, rmSync } from 'fs';
import { join } from 'path';
import { tmpdir } from 'os';
import {
charSimilarity,
wordRecall,
wer,
htmlToText,
runHtmlModality,
runAudioModality,
runPdfModality,
runCat11,
} from '../../eval/runner/cat11-multimodal.ts';
function tmpFixturesRoot(): string {
const dir = join(tmpdir(), `cat11-test-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`);
mkdirSync(dir, { recursive: true });
return dir;
}
// ─── Pure math helpers ────────────────────────────────────────────────
describe('charSimilarity', () => {
test('identical strings → 1', () => {
expect(charSimilarity('hello world', 'hello world')).toBe(1);
});
test('empty vs non-empty → 0', () => {
expect(charSimilarity('', 'hello')).toBe(0);
expect(charSimilarity('hello', '')).toBe(0);
});
test('both empty → 1', () => {
expect(charSimilarity('', '')).toBe(1);
});
test('one-char substitution', () => {
// 'cat' vs 'bat' = 1 edit over max 3 chars = 1 - 1/3 = 0.667
expect(charSimilarity('cat', 'bat')).toBeCloseTo(2 / 3, 6);
});
});
describe('wordRecall', () => {
test('all canonical words present → 1', () => {
expect(wordRecall('the quick brown fox', 'the lazy fox jumped over the quick brown dog')).toBe(1);
});
test('none present → 0', () => {
expect(wordRecall('one two three', 'alpha beta gamma')).toBe(0);
});
test('empty canonical → 1 (trivially satisfied)', () => {
expect(wordRecall('', 'anything')).toBe(1);
});
test('multiset semantics: three "the" vs one "the" → 1/3', () => {
expect(wordRecall('the the the', 'the cat')).toBeCloseTo(1 / 3, 6);
});
test('case-insensitive + strips punctuation', () => {
expect(wordRecall('Hello, world!', 'hello world')).toBe(1);
});
});
describe('wer', () => {
test('perfect transcription → 0', () => {
expect(wer('hello world', 'hello world')).toBe(0);
});
test('one word wrong → 0.5 on 2-word reference', () => {
expect(wer('hello world', 'hello there')).toBe(0.5);
});
test('empty reference vs empty hypothesis → 0', () => {
expect(wer('', '')).toBe(0);
});
test('empty reference + non-empty hypothesis → 1', () => {
expect(wer('', 'anything')).toBe(1);
});
test('completely wrong → 1', () => {
expect(wer('one two three', 'alpha beta gamma')).toBe(1);
});
});
describe('htmlToText', () => {
test('strips tags', () => {
expect(htmlToText('<p>hello <b>world</b></p>')).toBe('hello world');
});
test('drops script + style contents entirely', () => {
const html = '<html><head><style>body{}</style><script>alert(1)</script></head><body>visible</body></html>';
const out = htmlToText(html);
expect(out).toContain('visible');
expect(out).not.toContain('alert');
expect(out).not.toContain('body{}');
});
test('decodes common entities', () => {
expect(htmlToText('&lt;tag&gt; &amp; &quot;quote&quot;')).toContain('<tag> & "quote"');
});
test('collapses whitespace', () => {
expect(htmlToText('<p>a\n\nb \t c</p>')).toBe('a b c');
});
});
// ─── runHtmlModality (real filesystem + real manifest) ────────────────
describe('runHtmlModality', () => {
let root: string;
beforeEach(() => {
root = tmpFixturesRoot();
});
afterEach(() => {
rmSync(root, { recursive: true, force: true });
});
test('skips gracefully when manifest missing', async () => {
const result = await runHtmlModality(root);
expect(result.skipped).toBe(true);
expect(result.skip_reason).toContain('manifest');
});
test('scores real HTML fixture against canonical', async () => {
writeFileSync(join(root, 'page.html'), '<html><body><p>hello world</p></body></html>');
writeFileSync(join(root, 'page.txt'), 'hello world');
writeFileSync(
join(root, 'fixtures.json'),
JSON.stringify({
version: 1,
license: 'CC-BY-SA',
items: [
{
name: 'page',
path: 'page.html',
canonical_path: 'page.txt',
sha256: 'unused-in-test',
},
],
}),
);
const result = await runHtmlModality(root);
expect(result.skipped).toBe(false);
expect(result.items).toBe(1);
expect(result.items_attempted).toBe(1);
expect(result.mean_metric).toBe(1); // canonical "hello world" recovered exactly
expect(result.per_item[0].name).toBe('page');
});
test('missing fixture file produces an error entry (not a crash)', async () => {
writeFileSync(
join(root, 'fixtures.json'),
JSON.stringify({
version: 1,
license: 'CC-BY-SA',
items: [{ name: 'ghost', path: 'ghost.html', canonical_path: 'ghost.txt', sha256: 'x' }],
}),
);
const result = await runHtmlModality(root);
expect(result.skipped).toBe(false);
expect(result.per_item[0].error).toContain('missing');
expect(result.items_attempted).toBe(0);
});
});
// ─── runAudioModality ─────────────────────────────────────────────────
describe('runAudioModality', () => {
let root: string;
const originalGroq = process.env.GROQ_API_KEY;
const originalOpenAI = process.env.OPENAI_API_KEY;
beforeEach(() => {
root = tmpFixturesRoot();
delete process.env.GROQ_API_KEY;
delete process.env.OPENAI_API_KEY;
});
afterEach(() => {
rmSync(root, { recursive: true, force: true });
if (originalGroq !== undefined) process.env.GROQ_API_KEY = originalGroq;
if (originalOpenAI !== undefined) process.env.OPENAI_API_KEY = originalOpenAI;
});
test('skips gracefully when manifest missing', async () => {
const result = await runAudioModality(root);
expect(result.skipped).toBe(true);
expect(result.skip_reason).toContain('manifest');
});
test('skips gracefully when manifest present but no API key + no stub', async () => {
writeFileSync(
join(root, 'fixtures.json'),
JSON.stringify({
version: 1,
license: 'CC0',
items: [{ name: 'clip', path: 'clip.mp3', canonical_path: 'clip.txt', sha256: 'x' }],
}),
);
const result = await runAudioModality(root);
expect(result.skipped).toBe(true);
expect(result.skip_reason).toContain('GROQ_API_KEY');
});
test('uses injected transcriber stub and computes WER', async () => {
writeFileSync(join(root, 'clip.mp3'), 'fake audio bytes');
writeFileSync(join(root, 'clip.txt'), 'hello world this is a test');
writeFileSync(
join(root, 'fixtures.json'),
JSON.stringify({
version: 1,
license: 'CC0',
items: [{ name: 'clip', path: 'clip.mp3', canonical_path: 'clip.txt', sha256: 'x' }],
}),
);
const stubTranscribe = async () => ({ text: 'hello world this is a test', provider: 'stub' });
const result = await runAudioModality(root, { transcribe: stubTranscribe });
expect(result.skipped).toBe(false);
expect(result.mean_metric).toBe(0); // perfect transcription → WER 0
});
test('stub that returns wrong transcription produces non-zero WER', async () => {
writeFileSync(join(root, 'clip.mp3'), 'bytes');
writeFileSync(join(root, 'clip.txt'), 'one two three four');
writeFileSync(
join(root, 'fixtures.json'),
JSON.stringify({
version: 1,
license: 'CC0',
items: [{ name: 'clip', path: 'clip.mp3', canonical_path: 'clip.txt', sha256: 'x' }],
}),
);
const stub = async () => ({ text: 'one two three five', provider: 'stub' });
const result = await runAudioModality(root, { transcribe: stub });
expect(result.skipped).toBe(false);
expect(result.mean_metric).toBe(0.25); // 1 word wrong / 4 words = 0.25
});
});
// ─── runPdfModality ───────────────────────────────────────────────────
describe('runPdfModality', () => {
let root: string;
beforeEach(() => { root = tmpFixturesRoot(); });
afterEach(() => { rmSync(root, { recursive: true, force: true }); });
test('skips gracefully when manifest missing', async () => {
const result = await runPdfModality(root);
expect(result.skipped).toBe(true);
expect(result.skip_reason).toContain('manifest');
});
test('missing PDF file produces error entry (not crash)', async () => {
writeFileSync(
join(root, 'fixtures.json'),
JSON.stringify({
version: 1,
license: 'CC-BY',
items: [{ name: 'paper', path: 'ghost.pdf', canonical_path: 'ghost.txt', sha256: 'x' }],
}),
);
const result = await runPdfModality(root);
expect(result.skipped).toBe(false);
expect(result.per_item[0].error).toContain('missing');
});
});
// ─── runCat11 full dispatch ───────────────────────────────────────────
describe('runCat11', () => {
let root: string;
beforeEach(() => { root = tmpFixturesRoot(); });
afterEach(() => { rmSync(root, { recursive: true, force: true }); });
test('returns skipped verdict when no modality has fixtures', async () => {
const report = await runCat11({ fixturesRoot: root });
expect(report.verdict).toBe('skipped');
expect(report.results.pdf.skipped).toBe(true);
expect(report.results.html.skipped).toBe(true);
expect(report.results.audio.skipped).toBe(true);
});
test('returns baseline_only when at least one modality ran', async () => {
// Set up an HTML fixture so at least one modality runs
mkdirSync(join(root, 'html'), { recursive: true });
writeFileSync(join(root, 'html/p.html'), '<body>hello</body>');
writeFileSync(join(root, 'html/p.txt'), 'hello');
writeFileSync(
join(root, 'html/fixtures.json'),
JSON.stringify({
version: 1,
license: 'CC-BY-SA',
items: [{ name: 'p', path: 'p.html', canonical_path: 'p.txt', sha256: 'x' }],
}),
);
const report = await runCat11({ fixturesRoot: root });
expect(report.verdict).toBe('baseline_only');
expect(report.results.html.skipped).toBe(false);
});
});
Reference in New Issue View Git Blame Copy Permalink