diff --git a/eval/runner/adversarial-injections.ts b/eval/runner/adversarial-injections.ts new file mode 100644 index 000000000..3129c2cd1 --- /dev/null +++ b/eval/runner/adversarial-injections.ts @@ -0,0 +1,316 @@ +/** + * Adversarial injection kinds shared by Cat 10 (adversarial.ts — 22 hand- + * crafted edge cases) and Cat 6 (cat6-prose-scale.ts — 500+ prose-page + * variants). Each kind is a deterministic transform from a base page + * content string to a modified content string + a gold-delta description + * of what the extractor's output should look like. + * + * The module lives alongside adversarial.ts but is NEW code — adversarial.ts + * itself is a static case list, not a reusable transform engine. That's why + * the original plan said "extract the engine from adversarial.ts" but codex + * caught the overstatement: there's nothing to extract, we're building + * the engine fresh here. + * + * Each injection has a `kind` tag that flows into `eval/data/world-v1- + * adversarial/_adversarial/{slug}.json` metadata so the Cat 6 scorer can + * compute per-kind precision / recall / false-positive rate. + */ + +export type InjectionKind = + | 'code_fence_leak' // fake [X](people/fake-slug) inside ``` fences — must NOT extract + | 'inline_code_slug' // `people/x` in backticks — must NOT extract + | 'substring_collision' // "SamAI" near real person "sam" — exactly one link + | 'ambiguous_role' // "works with" vs "works at" — downgrade to `mentions` + | 'prose_only_mention' // strip the [X](people/slug) syntax; bare name remains — `mentions` only + | 'multi_entity_sentence';// 4+ entities in one clause — all N links extracted + +export interface EntityRef { + /** Slug matching the one-slash validator regex. */ + slug: string; + /** Display name, used when building markdown links. */ + name: string; +} + +export interface InjectionInput { + /** The base content to inject into. Usually compiled_truth from a world-v1 page. */ + content: string; + /** Seed for deterministic variant selection. */ + seed: number; + /** Available entity references for this corpus (sampled to build transforms). */ + refs: EntityRef[]; + /** Specific entities to use (overrides `refs` sampling; useful for tests). */ + forcedRefs?: EntityRef[]; +} + +export interface InjectionResult { + /** Transformed content with the injection applied. */ + content: string; + /** Structured description of what the extractor should and should NOT produce. */ + goldDelta: GoldDelta; +} + +export interface GoldDelta { + /** + * Links that MUST NOT appear in extraction output. Used by Cat 6's + * `false_positive_rate` metric. For code_fence_leak / inline_code_slug + * this lists the fake slugs inside fences; for substring_collision the + * near-miss match; etc. + */ + must_not_extract: Array<{ slug: string; reason: string }>; + /** + * Links that MUST appear in extraction output (exact match). Used by + * Cat 6 recall + per-kind recall. Empty when the injection is purely + * negative (e.g., isolated code_fence_leak test). + */ + must_extract: Array<{ slug: string; type: string; reason: string }>; + /** Human-readable note for the scorer's error reporting. */ + note: string; +} + +// ─── Seeded PRNG (Mulberry32, matches amara-life.ts) ────────────────── + +function createRng(seed: number): () => number { + let state = seed >>> 0; + return () => { + state = (state + 0x6D2B79F5) | 0; + let t = Math.imul(state ^ (state >>> 15), 1 | state); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +function pick(rng: () => number, xs: T[]): T { + if (xs.length === 0) throw new Error('pick: cannot pick from empty array'); + return xs[Math.floor(rng() * xs.length)]; +} + +// ─── Individual injections ──────────────────────────────────────────── + +export function injectCodeFenceLeak(input: InjectionInput): InjectionResult { + const rng = createRng(input.seed); + const realTarget = pick(rng, input.forcedRefs ?? input.refs); + const fakeSlug = `people/fake-${input.seed % 10000}`; + + const fenceBlock = [ + '```', + `const example = "${fakeSlug}";`, + `// See also [FakeName](${fakeSlug}) — should NOT extract from inside a fence`, + '```', + ].join('\n'); + + // Real mention (OUTSIDE the fence) so the baseline link is still there. + const realMention = `See also [${realTarget.name}](${realTarget.slug}) for more.`; + + const content = `${input.content}\n\n${fenceBlock}\n\n${realMention}`; + + return { + content, + goldDelta: { + must_not_extract: [ + { + slug: fakeSlug, + reason: 'code_fence_leak: slug appears inside triple-backtick fence', + }, + ], + must_extract: [ + { + slug: realTarget.slug, + type: 'mentions', + reason: 'code_fence_leak: real mention outside the fence should still extract', + }, + ], + note: `Injected fake slug ${fakeSlug} inside code fence + real mention ${realTarget.slug} outside.`, + }, + }; +} + +export function injectInlineCodeSlug(input: InjectionInput): InjectionResult { + const rng = createRng(input.seed); + const realTarget = pick(rng, input.forcedRefs ?? input.refs); + const fakeSlug = `people/inline-fake-${input.seed % 10000}`; + + const content = `${input.content}\n\nUse the \`${fakeSlug}\` notation in code. Real ref: [${realTarget.name}](${realTarget.slug}).`; + + return { + content, + goldDelta: { + must_not_extract: [ + { + slug: fakeSlug, + reason: 'inline_code_slug: slug wrapped in single-backtick inline code', + }, + ], + must_extract: [ + { slug: realTarget.slug, type: 'mentions', reason: 'inline_code_slug: real mention outside inline code' }, + ], + note: `Injected fake slug ${fakeSlug} in inline code + real mention ${realTarget.slug}.`, + }, + }; +} + +export function injectSubstringCollision(input: InjectionInput): InjectionResult { + const rng = createRng(input.seed); + const realTarget = pick(rng, input.forcedRefs ?? input.refs); + // Create a substring collision: if realTarget.name is "Sam", the collision + // is "SamAI". The extractor should match ONLY the [Sam](slug) reference, + // NOT "SamAI" appearing as prose. + const baseName = realTarget.name.split(' ')[0]; + const collisionWord = `${baseName}AI`; + + const content = `${input.content}\n\nThe ${collisionWord} initiative (unrelated) was launched last quarter. In parallel, [${realTarget.name}](${realTarget.slug}) continued their work on climate tooling.`; + + return { + content, + goldDelta: { + must_not_extract: [ + { + slug: `people/${baseName.toLowerCase()}ai`, + reason: `substring_collision: "${collisionWord}" must not be auto-linked to a people/ slug`, + }, + ], + must_extract: [ + { slug: realTarget.slug, type: 'mentions', reason: 'substring_collision: real markdown link should extract' }, + ], + note: `Injected substring collision "${collisionWord}" near real mention ${realTarget.slug}.`, + }, + }; +} + +export function injectAmbiguousRole(input: InjectionInput): InjectionResult { + const rng = createRng(input.seed); + const realTarget = pick(rng, input.forcedRefs ?? input.refs); + // Replace "works at" → "works with" if present, else append a "works with" + // sentence. "works with" should downgrade to `mentions`, not `works_at`. + const hasWorksAt = /works at/i.test(input.content); + const content = hasWorksAt + ? input.content.replace(/works at/gi, 'works with') + : `${input.content}\n\nShe regularly works with [${realTarget.name}](${realTarget.slug}) on quarterly reviews.`; + + return { + content, + goldDelta: { + must_not_extract: [], // The slug IS extracted; we just want type = mentions, not works_at + must_extract: [ + { + slug: realTarget.slug, + type: 'mentions', + reason: 'ambiguous_role: "works with" is loose enough that type must downgrade from works_at to mentions', + }, + ], + note: `Injected "works with" phrasing for ${realTarget.slug} (must not upgrade to works_at).`, + }, + }; +} + +export function injectProseOnlyMention(input: InjectionInput): InjectionResult { + const rng = createRng(input.seed); + const realTarget = pick(rng, input.forcedRefs ?? input.refs); + // Strip markdown link syntax around the target's name — leave the bare name + // in prose. Extractor should only produce a `mentions` link (not `founded` + // or any typed relation), because the evidence is just prose co-occurrence. + const nameRe = new RegExp( + `\\[${escapeRegex(realTarget.name)}\\]\\(${escapeRegex(realTarget.slug)}\\)`, + 'g', + ); + const stripped = input.content.replace(nameRe, realTarget.name); + const content = + stripped.length === input.content.length + ? `${input.content}\n\n${realTarget.name} has been a familiar figure in the industry for years.` + : stripped; + + return { + content, + goldDelta: { + must_not_extract: [], + must_extract: [ + { + slug: realTarget.slug, + type: 'mentions', + reason: 'prose_only_mention: bare name in prose resolves to mentions, not typed relation', + }, + ], + note: `Stripped markdown link around ${realTarget.name}, leaving bare prose mention only.`, + }, + }; +} + +export function injectMultiEntitySentence(input: InjectionInput): InjectionResult { + const rng = createRng(input.seed); + const availableRefs = input.forcedRefs ?? input.refs; + const n = Math.min(5, availableRefs.length); + if (n < 4) { + // Degenerate — not enough refs to build a multi-entity sentence. + return { + content: input.content, + goldDelta: { + must_not_extract: [], + must_extract: [], + note: 'multi_entity_sentence: skipped (fewer than 4 refs available)', + }, + }; + } + const picks: EntityRef[] = []; + const seen = new Set(); + // Deterministic pick of n unique refs + let attempts = 0; + while (picks.length < n && attempts < n * 10) { + const ref = pick(rng, availableRefs); + if (!seen.has(ref.slug)) { + picks.push(ref); + seen.add(ref.slug); + } + attempts++; + } + + const mdLinks = picks.map(r => `[${r.name}](${r.slug})`); + const clause = + `At the most recent Halfway partners meeting, ` + + `${mdLinks.slice(0, -1).join(', ')}, and ${mdLinks[mdLinks.length - 1]} all discussed the quarterly outlook.`; + + const content = `${input.content}\n\n${clause}`; + + return { + content, + goldDelta: { + must_not_extract: [], + must_extract: picks.map(r => ({ + slug: r.slug, + type: 'mentions', + reason: `multi_entity_sentence: all ${n} entities in a packed clause should extract`, + })), + note: `Injected packed clause with ${n} entities: ${picks.map(r => r.slug).join(', ')}.`, + }, + }; +} + +// ─── Dispatcher ────────────────────────────────────────────────────── + +export const ALL_INJECTION_KINDS: readonly InjectionKind[] = [ + 'code_fence_leak', + 'inline_code_slug', + 'substring_collision', + 'ambiguous_role', + 'prose_only_mention', + 'multi_entity_sentence', +] as const; + +export function applyInjection(kind: InjectionKind, input: InjectionInput): InjectionResult { + switch (kind) { + case 'code_fence_leak': + return injectCodeFenceLeak(input); + case 'inline_code_slug': + return injectInlineCodeSlug(input); + case 'substring_collision': + return injectSubstringCollision(input); + case 'ambiguous_role': + return injectAmbiguousRole(input); + case 'prose_only_mention': + return injectProseOnlyMention(input); + case 'multi_entity_sentence': + return injectMultiEntitySentence(input); + } +} + +function escapeRegex(s: string): string { + return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} diff --git a/eval/runner/cat11-multimodal.ts b/eval/runner/cat11-multimodal.ts new file mode 100644 index 000000000..34d36320d --- /dev/null +++ b/eval/runner/cat11-multimodal.ts @@ -0,0 +1,451 @@ +/** + * Cat 11 — Multi-modal ingestion fidelity. + * + * Measures text fidelity across three modalities: + * + * - **PDF** (10 arXiv papers, CC-licensed): char-level similarity vs + * canonical .txt + entity recall. Threshold (informational): >0.95. + * - **Audio** (5 Common Voice CC0 clips): WER vs canonical .txt. + * Opt-in via `GROQ_API_KEY` or `OPENAI_API_KEY`. Threshold: <0.15. + * - **HTML** (10 Wikipedia snapshots, CC-BY-SA): prose word-recall vs + * canonical .txt. Threshold: >0.80. + * + * Image OCR + video deferred to Cat 11 v2. XLSX also deferred (use authored + * markdown tables instead — codex flagged the xlsx parser dep as not worth + * adding for v1). + * + * Fixtures are NOT committed to the repo. They download on demand via + * `bun run eval:fetch-multimodal` using hash-verified manifests at + * `eval/data/multimodal//fixtures.json`. This keeps the + * repo lean while preserving reproducibility. + */ + +import { existsSync, readFileSync } from 'fs'; +import { join, resolve } from 'path'; +import { extractPdfText } from './loaders/pdf.ts'; + +// ─── Manifest types ────────────────────────────────────────────────── + +export interface FixtureManifest { + version: 1; + license: string; + items: Array<{ + name: string; + path: string; + canonical_path: string; + sha256: string; + /** For entity-recall scoring (PDF). Optional list of expected entity names. */ + entities?: string[]; + }>; +} + +// ─── Per-modality results ──────────────────────────────────────────── + +export interface ModalityResult { + modality: 'pdf' | 'audio' | 'html'; + items: number; + items_attempted: number; + /** Mean similarity / fidelity / recall for the modality. */ + mean_metric: number; + /** Per-item breakdown. */ + per_item: Array<{ + name: string; + metric: number; + /** Optional — e.g., number of entities matched (PDF) or word-count (HTML). */ + detail?: Record; + error?: string; + }>; + /** True if the modality was skipped (e.g., missing fixtures or API key). */ + skipped: boolean; + skip_reason?: string; +} + +export interface Cat11Report { + schema_version: 1; + ran_at: string; + results: Record<'pdf' | 'audio' | 'html', ModalityResult>; + verdict: 'pass' | 'fail' | 'baseline_only' | 'skipped'; +} + +// ─── Char-level similarity (Levenshtein-based) ─────────────────────── + +/** + * Char-level similarity using normalized Levenshtein edit distance. + * Returns 0..1 where 1 = exact match. O(n*m) memory — fine for PDF/HTML + * comparisons capped at ~100KB each. + */ +export function charSimilarity(a: string, b: string): number { + if (a === b) return 1; + if (a.length === 0 || b.length === 0) return 0; + const n = a.length; + const m = b.length; + // Use typed arrays for speed on larger strings + let prev = new Int32Array(m + 1); + let curr = new Int32Array(m + 1); + for (let j = 0; j <= m; j++) prev[j] = j; + for (let i = 1; i <= n; i++) { + curr[0] = i; + for (let j = 1; j <= m; j++) { + const cost = a.charCodeAt(i - 1) === b.charCodeAt(j - 1) ? 0 : 1; + curr[j] = Math.min( + prev[j] + 1, // deletion + curr[j - 1] + 1, // insertion + prev[j - 1] + cost, // substitution + ); + } + [prev, curr] = [curr, prev]; + } + const distance = prev[m]; + const maxLen = Math.max(n, m); + return 1 - distance / maxLen; +} + +// ─── Word-level recall (for HTML) ──────────────────────────────────── + +function normalizeWords(s: string): string[] { + return s + .toLowerCase() + .replace(/[^a-z0-9\s]+/g, ' ') + .split(/\s+/) + .filter(w => w.length > 0); +} + +/** + * Word recall: fraction of canonical words present in extracted text. + * Multiset semantics: if canonical has "the the the" and extracted has "the", + * recall is 1/3. + */ +export function wordRecall(canonical: string, extracted: string): number { + const canonicalWords = normalizeWords(canonical); + const extractedCounts = new Map(); + for (const w of normalizeWords(extracted)) { + extractedCounts.set(w, (extractedCounts.get(w) ?? 0) + 1); + } + if (canonicalWords.length === 0) return 1; + let hits = 0; + for (const w of canonicalWords) { + const count = extractedCounts.get(w) ?? 0; + if (count > 0) { + hits++; + extractedCounts.set(w, count - 1); + } + } + return hits / canonicalWords.length; +} + +// ─── WER (word error rate) for audio ───────────────────────────────── + +/** + * Word Error Rate: Levenshtein distance at word level / reference word count. + * Lower is better. 0.0 = perfect transcription. Typical whisper-v3 on clean + * English ≈ 0.05-0.10. + */ +export function wer(reference: string, hypothesis: string): number { + const ref = normalizeWords(reference); + const hyp = normalizeWords(hypothesis); + if (ref.length === 0) return hyp.length === 0 ? 0 : 1; + + const n = ref.length; + const m = hyp.length; + let prev = new Int32Array(m + 1); + let curr = new Int32Array(m + 1); + for (let j = 0; j <= m; j++) prev[j] = j; + for (let i = 1; i <= n; i++) { + curr[0] = i; + for (let j = 1; j <= m; j++) { + const cost = ref[i - 1] === hyp[j - 1] ? 0 : 1; + curr[j] = Math.min(prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + cost); + } + [prev, curr] = [curr, prev]; + } + return prev[m] / n; +} + +// ─── PDF modality ───────────────────────────────────────────────────── + +export async function runPdfModality( + fixturesDir: string = resolve(process.cwd(), 'eval/data/multimodal/pdf'), +): Promise { + const manifestPath = join(fixturesDir, 'fixtures.json'); + if (!existsSync(manifestPath)) { + return { + modality: 'pdf', + items: 0, + items_attempted: 0, + mean_metric: 0, + per_item: [], + skipped: true, + skip_reason: `No manifest at ${manifestPath}. Run \`bun run eval:fetch-multimodal\` first.`, + }; + } + const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) as FixtureManifest; + const perItem: ModalityResult['per_item'] = []; + let metricSum = 0; + let attempted = 0; + + for (const item of manifest.items) { + const pdfPath = join(fixturesDir, item.path); + const canonicalPath = join(fixturesDir, item.canonical_path); + if (!existsSync(pdfPath) || !existsSync(canonicalPath)) { + perItem.push({ + name: item.name, + metric: 0, + error: `missing fixture file(s) at ${item.path} / ${item.canonical_path}`, + }); + continue; + } + try { + const extracted = await extractPdfText(pdfPath); + const canonical = readFileSync(canonicalPath, 'utf8'); + const similarity = charSimilarity(canonical.trim(), extracted.text.trim()); + + // Entity recall if the manifest declares expected entities + let entityRecall: number | undefined; + if (item.entities && item.entities.length > 0) { + const hits = item.entities.filter(name => + extracted.text.toLowerCase().includes(name.toLowerCase()), + ).length; + entityRecall = hits / item.entities.length; + } + + perItem.push({ + name: item.name, + metric: similarity, + detail: { + num_pages: extracted.numPages, + canonical_chars: canonical.length, + extracted_chars: extracted.text.length, + ...(entityRecall !== undefined ? { entity_recall: entityRecall } : {}), + }, + }); + metricSum += similarity; + attempted++; + } catch (err) { + perItem.push({ name: item.name, metric: 0, error: String(err) }); + } + } + + return { + modality: 'pdf', + items: manifest.items.length, + items_attempted: attempted, + mean_metric: attempted === 0 ? 0 : metricSum / attempted, + per_item: perItem, + skipped: false, + }; +} + +// ─── HTML modality ──────────────────────────────────────────────────── + +/** + * Strip HTML tags + common noise, collapse whitespace. Not a full HTML + * parser — intentionally simple to make the metric stable across HTML + * variations. + */ +export function htmlToText(html: string): string { + return html + .replace(//gi, ' ') + .replace(//gi, ' ') + .replace(/<[^>]+>/g, ' ') + .replace(/ /gi, ' ') + .replace(/&/gi, '&') + .replace(/</gi, '<') + .replace(/>/gi, '>') + .replace(/"/gi, '"') + .replace(/\s+/g, ' ') + .trim(); +} + +export async function runHtmlModality( + fixturesDir: string = resolve(process.cwd(), 'eval/data/multimodal/html'), +): Promise { + const manifestPath = join(fixturesDir, 'fixtures.json'); + if (!existsSync(manifestPath)) { + return { + modality: 'html', + items: 0, + items_attempted: 0, + mean_metric: 0, + per_item: [], + skipped: true, + skip_reason: `No manifest at ${manifestPath}.`, + }; + } + const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) as FixtureManifest; + const perItem: ModalityResult['per_item'] = []; + let metricSum = 0; + let attempted = 0; + + for (const item of manifest.items) { + const htmlPath = join(fixturesDir, item.path); + const canonicalPath = join(fixturesDir, item.canonical_path); + if (!existsSync(htmlPath) || !existsSync(canonicalPath)) { + perItem.push({ + name: item.name, + metric: 0, + error: `missing fixture file(s)`, + }); + continue; + } + try { + const html = readFileSync(htmlPath, 'utf8'); + const canonical = readFileSync(canonicalPath, 'utf8'); + const extracted = htmlToText(html); + const recall = wordRecall(canonical, extracted); + perItem.push({ + name: item.name, + metric: recall, + detail: { + canonical_chars: canonical.length, + extracted_chars: extracted.length, + }, + }); + metricSum += recall; + attempted++; + } catch (err) { + perItem.push({ name: item.name, metric: 0, error: String(err) }); + } + } + + return { + modality: 'html', + items: manifest.items.length, + items_attempted: attempted, + mean_metric: attempted === 0 ? 0 : metricSum / attempted, + per_item: perItem, + skipped: false, + }; +} + +// ─── Audio modality ────────────────────────────────────────────────── + +export interface AudioRunOpts { + /** Injectable transcriber for tests. Default uses src/core/transcription.ts. */ + transcribe?: (audioPath: string) => Promise<{ text: string; provider: string }>; +} + +export async function runAudioModality( + fixturesDir: string = resolve(process.cwd(), 'eval/data/multimodal/audio'), + opts: AudioRunOpts = {}, +): Promise { + const manifestPath = join(fixturesDir, 'fixtures.json'); + if (!existsSync(manifestPath)) { + return { + modality: 'audio', + items: 0, + items_attempted: 0, + mean_metric: 0, + per_item: [], + skipped: true, + skip_reason: `No manifest at ${manifestPath}.`, + }; + } + + // Gate on API keys unless a test-injected transcriber is provided. + const hasApiKey = !!process.env.GROQ_API_KEY || !!process.env.OPENAI_API_KEY; + if (!opts.transcribe && !hasApiKey) { + return { + modality: 'audio', + items: 0, + items_attempted: 0, + mean_metric: 0, + per_item: [], + skipped: true, + skip_reason: 'Neither GROQ_API_KEY nor OPENAI_API_KEY is set; audio transcription requires one.', + }; + } + + const transcriber = opts.transcribe ?? (await loadDefaultTranscriber()); + + const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) as FixtureManifest; + const perItem: ModalityResult['per_item'] = []; + let metricSum = 0; + let attempted = 0; + + for (const item of manifest.items) { + const audioPath = join(fixturesDir, item.path); + const canonicalPath = join(fixturesDir, item.canonical_path); + if (!existsSync(audioPath) || !existsSync(canonicalPath)) { + perItem.push({ name: item.name, metric: 1, error: 'missing fixture file(s)' }); + continue; + } + try { + const { text, provider } = await transcriber(audioPath); + const canonical = readFileSync(canonicalPath, 'utf8'); + const errorRate = wer(canonical, text); + perItem.push({ + name: item.name, + metric: errorRate, + detail: { + canonical_words: normalizeWords(canonical).length, + transcribed_words: normalizeWords(text).length, + provider_length: provider.length, + }, + }); + metricSum += errorRate; + attempted++; + } catch (err) { + perItem.push({ name: item.name, metric: 1, error: String(err) }); + } + } + + return { + modality: 'audio', + items: manifest.items.length, + items_attempted: attempted, + mean_metric: attempted === 0 ? 0 : metricSum / attempted, + per_item: perItem, + skipped: false, + }; +} + +async function loadDefaultTranscriber(): Promise> { + // Lazy import src/core/transcription.ts so this file doesn't drag that + // import into module-load time. transcription.ts has env checks that + // throw if GROQ/OPENAI keys are absent. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const mod: any = await import('../../src/core/transcription.ts'); + const fn = mod.transcribe ?? mod.default; + if (typeof fn !== 'function') { + throw new Error('src/core/transcription.ts does not export a transcribe function'); + } + return async (audioPath: string) => { + const res = await fn(audioPath); + return { text: res.text ?? '', provider: res.provider ?? 'unknown' }; + }; +} + +// ─── Runner entry ───────────────────────────────────────────────────── + +export interface RunCat11Options { + fixturesRoot?: string; + /** Override the audio transcriber (test injection). */ + transcribeAudio?: AudioRunOpts['transcribe']; +} + +export async function runCat11(opts: RunCat11Options = {}): Promise { + const root = opts.fixturesRoot ?? resolve(process.cwd(), 'eval/data/multimodal'); + const [pdf, html, audio] = await Promise.all([ + runPdfModality(join(root, 'pdf')), + runHtmlModality(join(root, 'html')), + runAudioModality(join(root, 'audio'), { transcribe: opts.transcribeAudio }), + ]); + + const anyRan = !pdf.skipped || !html.skipped || !audio.skipped; + + return { + schema_version: 1, + ran_at: new Date().toISOString(), + results: { pdf, html, audio }, + verdict: anyRan ? 'baseline_only' : 'skipped', + }; +} + +if (import.meta.main) { + runCat11() + .then(report => console.log(JSON.stringify(report, null, 2))) + .catch(err => { + console.error(err); + process.exit(1); + }); +} diff --git a/eval/runner/cat6-prose-scale.ts b/eval/runner/cat6-prose-scale.ts new file mode 100644 index 000000000..e3a3764ed --- /dev/null +++ b/eval/runner/cat6-prose-scale.ts @@ -0,0 +1,402 @@ +/** + * Cat 6 — Auto-link Precision under Prose (at scale). + * + * Extends Cat 10 (adversarial.ts, 22 hand-crafted edge cases) to N prose-heavy + * page variants generated by applying the 6 injections from + * `adversarial-injections.ts` to world-v1 base pages. + * + * **v1 is baseline-only** (no gating threshold). Codex flagged that + * `type_accuracy > 0.80` is unreachable given current residuals + * (`works_at` 58%, `advises` 41% per TODOS.md). Cat 6 in v1 produces a + * numeric baseline the scorecard records; regression guard is + * "score must not drop below committed first baseline." Threshold gating + * returns after the extractor uplift TODO lands. + * + * Metrics: + * - link_precision, link_recall, link_f1 (overall) + * - per-injection-kind: precision, recall, false_positive_rate + * - code_fence_leak_rate = leaked_fence_links / total_fence_cases + * - substring_fp_rate = collided_substring_links / total_collision_cases + * - pages_with_links_coverage (fraction of variants producing ≥1 link) + * - mean_links_per_page + */ + +import { readdirSync, readFileSync } from 'fs'; +import { join, resolve } from 'path'; +import { + ALL_INJECTION_KINDS, + applyInjection, + type EntityRef, + type GoldDelta, + type InjectionKind, +} from './adversarial-injections.ts'; +import { extractPageLinks, type SlugResolver } from '../../src/core/link-extraction.ts'; +import type { PageType } from '../../src/core/types.ts'; + +// ─── Types ──────────────────────────────────────────────────────────── + +export interface VariantCase { + /** Unique id for this variant, e.g. "people/alice-v0-code_fence_leak". */ + variantId: string; + /** Base slug the variant was derived from. */ + baseSlug: string; + baseType: PageType; + /** Injection applied to the base content. */ + kind: InjectionKind; + /** Transformed content (what the extractor sees). */ + content: string; + /** Gold delta from the injection. */ + goldDelta: GoldDelta; +} + +export interface ExtractedLink { + targetSlug: string; + linkType: string; +} + +export interface VariantResult { + variantId: string; + kind: InjectionKind; + extracted: ExtractedLink[]; + /** Slugs that appeared in extraction output but were marked must_not_extract. */ + false_positives: Array<{ slug: string; reason: string }>; + /** Slugs from must_extract that DID NOT appear in extraction output. */ + missed: Array<{ slug: string; type: string; reason: string }>; + /** Slugs from must_extract that DID appear (regardless of type match). */ + matched: number; +} + +export interface PerKindReport { + kind: InjectionKind; + variants: number; + total_must_extract: number; + total_matched: number; + total_false_positives: number; + precision: number; + recall: number; + false_positive_rate: number; +} + +export interface Cat6Report { + schema_version: 1; + ran_at: string; + variants: number; + corpus_id: string; + per_kind: PerKindReport[]; + overall: { + link_precision: number; + link_recall: number; + link_f1: number; + code_fence_leak_rate: number; + substring_fp_rate: number; + pages_with_links_coverage: number; + mean_links_per_page: number; + }; + verdict: 'baseline_only'; +} + +// ─── Corpus loading ────────────────────────────────────────────────── + +export interface BasePage { + slug: string; + type: PageType; + title: string; + content: string; +} + +/** + * Minimal loader for `eval/data/world-v1` JSON shards. Each shard has shape + * `{slug, type, title, compiled_truth, timeline, _facts}`. We concatenate + * compiled_truth + timeline as the extractor sees them. + */ +export function loadWorldV1( + corpusDir: string = resolve(process.cwd(), 'eval/data/world-v1'), +): BasePage[] { + const files = readdirSync(corpusDir).filter( + f => f.endsWith('.json') && !f.startsWith('_'), + ); + const pages: BasePage[] = []; + for (const f of files) { + const raw = readFileSync(join(corpusDir, f), 'utf8'); + const shard = JSON.parse(raw) as { + slug: string; + type: PageType; + title: string; + compiled_truth: string; + timeline?: string; + }; + pages.push({ + slug: shard.slug, + type: shard.type, + title: shard.title, + content: `${shard.compiled_truth}\n\n${shard.timeline ?? ''}`, + }); + } + return pages; +} + +// ─── Variant generation ────────────────────────────────────────────── + +function pagesToRefs(pages: BasePage[]): EntityRef[] { + return pages + .filter(p => p.type === 'person' || p.type === 'company') + .map(p => ({ slug: p.slug, name: p.title })); +} + +export interface VariantGenerationOpts { + /** Max variants per injection kind. Default 50 → 50 × 6 = 300 variants total. */ + perKind?: number; + /** Seed offset; each variant computes seed = baseSeed + variantIdx. Default 1000. */ + baseSeed?: number; + /** Optional set of base-page types to include. Default ['person', 'company']. */ + includeTypes?: PageType[]; +} + +/** + * Generate variant cases deterministically from a base corpus. Variants are + * spread across all 6 injection kinds; seed determinism means re-running + * against the same corpus produces identical variants. + */ +export function generateVariants( + pages: BasePage[], + opts: VariantGenerationOpts = {}, +): VariantCase[] { + const perKind = opts.perKind ?? 50; + const baseSeed = opts.baseSeed ?? 1000; + const includeTypes = opts.includeTypes ?? (['person', 'company'] as PageType[]); + const validPages = pages.filter(p => includeTypes.includes(p.type)); + if (validPages.length === 0) return []; + + const refs = pagesToRefs(pages); + + const variants: VariantCase[] = []; + let variantIdx = 0; + + for (const kind of ALL_INJECTION_KINDS) { + for (let i = 0; i < perKind; i++) { + // Deterministic base page pick: step through the corpus in order. + const base = validPages[(variantIdx + i) % validPages.length]; + const seed = baseSeed + variantIdx; + const res = applyInjection(kind, { content: base.content, seed, refs }); + variants.push({ + variantId: `${base.slug}-v${i}-${kind}`, + baseSlug: base.slug, + baseType: base.type, + kind, + content: res.content, + goldDelta: res.goldDelta, + }); + variantIdx++; + } + } + + return variants; +} + +// ─── Extraction + scoring ───────────────────────────────────────────── + +/** + * Minimal slug resolver: accepts only slugs present in the corpus. Returns + * `slug` unchanged if known, `null` otherwise. This matches Cat 6 semantics + * because the adversarial transforms never introduce NEW valid target slugs + * — any "link" the extractor produces must resolve to something real. + */ +export function makeCorpusResolver(pages: BasePage[]): SlugResolver { + const known = new Set(pages.map(p => p.slug)); + return { + resolve: async (name: string) => { + // Exact slug match — bare-name resolution is out of scope for Cat 6 scoring. + if (known.has(name)) return name; + return null; + }, + }; +} + +async function extractOne( + variant: VariantCase, + resolver: SlugResolver, +): Promise { + const res = await extractPageLinks( + variant.baseSlug, + variant.content, + {}, // frontmatter unused for Cat 6 prose tests + variant.baseType, + resolver, + ); + return res.candidates + .filter(c => c.targetSlug) + .map(c => ({ targetSlug: c.targetSlug, linkType: c.linkType ?? 'mentions' })); +} + +export async function scoreVariant( + variant: VariantCase, + resolver: SlugResolver, +): Promise { + const extracted = await extractOne(variant, resolver); + const extractedSlugs = new Set(extracted.map(e => e.targetSlug)); + + const false_positives: Array<{ slug: string; reason: string }> = []; + for (const must_not of variant.goldDelta.must_not_extract) { + if (extractedSlugs.has(must_not.slug)) { + false_positives.push(must_not); + } + } + + const missed: Array<{ slug: string; type: string; reason: string }> = []; + let matched = 0; + for (const must of variant.goldDelta.must_extract) { + if (extractedSlugs.has(must.slug)) { + matched++; + } else { + missed.push(must); + } + } + + return { + variantId: variant.variantId, + kind: variant.kind, + extracted, + false_positives, + missed, + matched, + }; +} + +// ─── Aggregation ────────────────────────────────────────────────────── + +function precision(matched: number, extracted: number): number { + return extracted === 0 ? 0 : matched / extracted; +} + +function recall(matched: number, expected: number): number { + return expected === 0 ? 1 : matched / expected; +} + +function f1(p: number, r: number): number { + if (p + r === 0) return 0; + return (2 * p * r) / (p + r); +} + +export function aggregate(variants: VariantCase[], results: VariantResult[]): Cat6Report { + const byKind = new Map(); + for (let i = 0; i < variants.length; i++) { + const v = variants[i]; + const r = results[i]; + if (!byKind.has(v.kind)) byKind.set(v.kind, { variants: [], results: [] }); + byKind.get(v.kind)!.variants.push(v); + byKind.get(v.kind)!.results.push(r); + } + + const per_kind: PerKindReport[] = []; + + let totalMust = 0; + let totalMatched = 0; + let totalExtracted = 0; + let totalFp = 0; + let totalFenceCases = 0; + let totalFenceLeaks = 0; + let totalCollisionCases = 0; + let totalCollisionFps = 0; + let variantsWithLinks = 0; + + for (const [kind, bucket] of byKind) { + let kindMust = 0; + let kindMatched = 0; + let kindExtracted = 0; + let kindFp = 0; + + for (let i = 0; i < bucket.variants.length; i++) { + const v = bucket.variants[i]; + const r = bucket.results[i]; + kindMust += v.goldDelta.must_extract.length; + kindMatched += r.matched; + kindExtracted += r.extracted.length; + kindFp += r.false_positives.length; + if (r.extracted.length > 0) variantsWithLinks++; + + if (kind === 'code_fence_leak') { + totalFenceCases += v.goldDelta.must_not_extract.length; + totalFenceLeaks += r.false_positives.length; + } + if (kind === 'substring_collision') { + totalCollisionCases += v.goldDelta.must_not_extract.length; + totalCollisionFps += r.false_positives.length; + } + } + + const p = precision(kindMatched, kindExtracted); + const r = recall(kindMatched, kindMust); + per_kind.push({ + kind, + variants: bucket.variants.length, + total_must_extract: kindMust, + total_matched: kindMatched, + total_false_positives: kindFp, + precision: p, + recall: r, + false_positive_rate: bucket.variants.length === 0 ? 0 : kindFp / bucket.variants.length, + }); + + totalMust += kindMust; + totalMatched += kindMatched; + totalExtracted += kindExtracted; + totalFp += kindFp; + } + + const overallP = precision(totalMatched, totalExtracted); + const overallR = recall(totalMatched, totalMust); + + return { + schema_version: 1, + ran_at: new Date().toISOString(), + variants: variants.length, + corpus_id: 'world-v1', + per_kind, + overall: { + link_precision: overallP, + link_recall: overallR, + link_f1: f1(overallP, overallR), + code_fence_leak_rate: totalFenceCases === 0 ? 0 : totalFenceLeaks / totalFenceCases, + substring_fp_rate: totalCollisionCases === 0 ? 0 : totalCollisionFps / totalCollisionCases, + pages_with_links_coverage: variants.length === 0 ? 0 : variantsWithLinks / variants.length, + mean_links_per_page: variants.length === 0 ? 0 : totalExtracted / variants.length, + }, + verdict: 'baseline_only', + }; +} + +// ─── Runner entry ───────────────────────────────────────────────────── + +export interface RunCat6Options { + corpusDir?: string; + perKind?: number; + baseSeed?: number; +} + +export async function runCat6(opts: RunCat6Options = {}): Promise { + const pages = loadWorldV1(opts.corpusDir); + const variants = generateVariants(pages, { + perKind: opts.perKind, + baseSeed: opts.baseSeed, + }); + const resolver = makeCorpusResolver(pages); + + const results: VariantResult[] = []; + for (const v of variants) { + results.push(await scoreVariant(v, resolver)); + } + return aggregate(variants, results); +} + +// ─── CLI smoke ──────────────────────────────────────────────────────── + +if (import.meta.main) { + runCat6() + .then(report => { + console.log(JSON.stringify(report, null, 2)); + }) + .catch(err => { + console.error(err); + process.exit(1); + }); +} diff --git a/test/eval/adversarial-injections.test.ts b/test/eval/adversarial-injections.test.ts new file mode 100644 index 000000000..723b418b5 --- /dev/null +++ b/test/eval/adversarial-injections.test.ts @@ -0,0 +1,181 @@ +/** + * adversarial-injections.ts tests — Day 6 of BrainBench v1 Complete. + * + * Covers every injection kind: + * - Each produces deterministic output under same seed + * - goldDelta.must_not_extract lists the adversarial slug for negative injections + * - goldDelta.must_extract lists real targets for positive assertions + * - applyInjection dispatcher returns the same output as the direct function + */ + +import { describe, test, expect } from 'bun:test'; +import { + applyInjection, + injectCodeFenceLeak, + injectInlineCodeSlug, + injectSubstringCollision, + injectAmbiguousRole, + injectProseOnlyMention, + injectMultiEntitySentence, + ALL_INJECTION_KINDS, + type EntityRef, +} from '../../eval/runner/adversarial-injections.ts'; + +const REFS: EntityRef[] = [ + { slug: 'people/amara-okafor', name: 'Amara Okafor' }, + { slug: 'people/jordan-park', name: 'Jordan Park' }, + { slug: 'people/mina-kapoor', name: 'Mina Kapoor' }, + { slug: 'people/sarah-chen', name: 'Sarah Chen' }, + { slug: 'people/priya-patel', name: 'Priya Patel' }, + { slug: 'companies/halfway-capital', name: 'Halfway Capital' }, + { slug: 'companies/novamind', name: 'NovaMind' }, +]; + +const BASE_CONTENT = 'This is a fictional biographical page. She is well-known in the climate deals space.'; + +// ─── Per-kind assertions ───────────────────────────────────────────── + +describe('injectCodeFenceLeak', () => { + test('wraps a fake slug inside triple-backtick fence', () => { + const res = injectCodeFenceLeak({ content: BASE_CONTENT, seed: 1, refs: REFS }); + expect(res.content).toContain('```'); + const fakeSlug = res.goldDelta.must_not_extract[0].slug; + expect(fakeSlug).toMatch(/^people\/fake-\d+$/); + expect(res.content).toContain(fakeSlug); + }); + + test('leaves a real mention outside the fence', () => { + const res = injectCodeFenceLeak({ content: BASE_CONTENT, seed: 1, refs: REFS }); + expect(res.goldDelta.must_extract.length).toBeGreaterThan(0); + const realSlug = res.goldDelta.must_extract[0].slug; + // Real slug should appear OUTSIDE the fence + const fenceEnd = res.content.lastIndexOf('```'); + const afterFence = res.content.slice(fenceEnd + 3); + expect(afterFence).toContain(realSlug); + }); + + test('deterministic under same seed', () => { + const a = injectCodeFenceLeak({ content: BASE_CONTENT, seed: 42, refs: REFS }); + const b = injectCodeFenceLeak({ content: BASE_CONTENT, seed: 42, refs: REFS }); + expect(a.content).toEqual(b.content); + }); +}); + +describe('injectInlineCodeSlug', () => { + test('wraps slug in single-backtick inline code', () => { + const res = injectInlineCodeSlug({ content: BASE_CONTENT, seed: 1, refs: REFS }); + const fakeSlug = res.goldDelta.must_not_extract[0].slug; + expect(res.content).toContain(`\`${fakeSlug}\``); + }); +}); + +describe('injectSubstringCollision', () => { + test('injects a "AI" substring near a real mention', () => { + const forced: EntityRef[] = [{ slug: 'people/sam', name: 'Sam' }]; + const res = injectSubstringCollision({ content: BASE_CONTENT, seed: 1, refs: REFS, forcedRefs: forced }); + expect(res.content).toContain('SamAI'); + expect(res.content).toContain('[Sam](people/sam)'); + expect(res.goldDelta.must_not_extract[0].slug).toBe('people/samai'); + }); +}); + +describe('injectAmbiguousRole', () => { + test('replaces "works at" with "works with" when present', () => { + const input = 'Alice works at [Acme](companies/acme).'; + const res = injectAmbiguousRole({ content: input, seed: 1, refs: REFS }); + expect(res.content).toContain('works with'); + expect(res.content).not.toContain('works at'); + }); + + test('appends a works-with sentence when source has no "works at"', () => { + const res = injectAmbiguousRole({ content: 'plain prose.', seed: 1, refs: REFS }); + expect(res.content).toContain('works with'); + }); + + test('goldDelta says type must be mentions, not works_at', () => { + const res = injectAmbiguousRole({ content: 'plain prose.', seed: 1, refs: REFS }); + expect(res.goldDelta.must_extract[0].type).toBe('mentions'); + }); +}); + +describe('injectProseOnlyMention', () => { + test('strips [name](slug) syntax leaving bare name', () => { + const forced: EntityRef[] = [{ slug: 'people/jordan-park', name: 'Jordan Park' }]; + const input = 'I met [Jordan Park](people/jordan-park) yesterday.'; + const res = injectProseOnlyMention({ content: input, seed: 1, refs: REFS, forcedRefs: forced }); + expect(res.content).not.toContain('[Jordan Park]'); + expect(res.content).not.toContain('(people/jordan-park)'); + expect(res.content).toContain('Jordan Park'); + }); + + test('goldDelta requires extraction as mentions', () => { + const forced: EntityRef[] = [{ slug: 'people/jordan-park', name: 'Jordan Park' }]; + const input = 'I met [Jordan Park](people/jordan-park) yesterday.'; + const res = injectProseOnlyMention({ content: input, seed: 1, refs: REFS, forcedRefs: forced }); + expect(res.goldDelta.must_extract[0]).toEqual({ + slug: 'people/jordan-park', + type: 'mentions', + reason: expect.any(String), + }); + }); +}); + +describe('injectMultiEntitySentence', () => { + test('packs 4+ entities into one clause', () => { + const res = injectMultiEntitySentence({ content: BASE_CONTENT, seed: 1, refs: REFS }); + expect(res.goldDelta.must_extract.length).toBeGreaterThanOrEqual(4); + // Every must_extract slug should appear in content as markdown link + for (const m of res.goldDelta.must_extract) { + expect(res.content).toContain(`](${m.slug})`); + } + }); + + test('skips gracefully when fewer than 4 refs are available', () => { + const res = injectMultiEntitySentence({ + content: BASE_CONTENT, + seed: 1, + refs: REFS.slice(0, 2), + }); + expect(res.goldDelta.must_extract.length).toBe(0); + expect(res.goldDelta.note).toContain('skipped'); + }); + + test('picks are unique (no duplicate slugs in one clause)', () => { + const res = injectMultiEntitySentence({ content: BASE_CONTENT, seed: 1, refs: REFS }); + const slugs = res.goldDelta.must_extract.map(m => m.slug); + expect(new Set(slugs).size).toBe(slugs.length); + }); +}); + +// ─── Dispatcher + kind coverage ────────────────────────────────────── + +describe('applyInjection dispatcher', () => { + test('ALL_INJECTION_KINDS lists exactly 6 kinds', () => { + expect(ALL_INJECTION_KINDS.length).toBe(6); + }); + + test('dispatcher produces same output as the direct function (code_fence_leak)', () => { + const direct = injectCodeFenceLeak({ content: BASE_CONTENT, seed: 5, refs: REFS }); + const viaDispatch = applyInjection('code_fence_leak', { content: BASE_CONTENT, seed: 5, refs: REFS }); + expect(viaDispatch.content).toBe(direct.content); + }); + + test('every kind produces a non-empty note in goldDelta', () => { + for (const kind of ALL_INJECTION_KINDS) { + const res = applyInjection(kind, { content: BASE_CONTENT, seed: 7, refs: REFS }); + expect(res.goldDelta.note.length).toBeGreaterThan(0); + } + }); + + test('every kind produces valid slug format in gold entries', () => { + const SLUG_RE = /^[a-z][a-z0-9-]*\/[a-z0-9][a-z0-9-]*$/; + for (const kind of ALL_INJECTION_KINDS) { + const res = applyInjection(kind, { content: BASE_CONTENT, seed: 9, refs: REFS }); + for (const m of res.goldDelta.must_extract) expect(m.slug).toMatch(SLUG_RE); + for (const m of res.goldDelta.must_not_extract) { + // Fake slugs are built to resemble real slug shape + expect(m.slug).toMatch(SLUG_RE); + } + } + }); +}); diff --git a/test/eval/cat11.test.ts b/test/eval/cat11.test.ts new file mode 100644 index 000000000..2f76dff2c --- /dev/null +++ b/test/eval/cat11.test.ts @@ -0,0 +1,316 @@ +/** + * cat11-multimodal.ts tests — Day 6 of BrainBench v1 Complete. + * + * Covers: + * - charSimilarity math: identical / empty / partial + * - wordRecall: multiset semantics, empty canonical + * - wer (word error rate): perfect / half-off / empty ref + * - htmlToText: strips script/style/tags, decodes common entities + * - runHtmlModality with a real fixture (tmpdir, hand-rolled HTML) + * - runAudioModality skips gracefully without API keys + manifest + * - runPdfModality skips gracefully without manifest + * - runAudioModality uses injected transcriber stub + */ + +import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; +import { mkdirSync, writeFileSync, rmSync } from 'fs'; +import { join } from 'path'; +import { tmpdir } from 'os'; +import { + charSimilarity, + wordRecall, + wer, + htmlToText, + runHtmlModality, + runAudioModality, + runPdfModality, + runCat11, +} from '../../eval/runner/cat11-multimodal.ts'; + +function tmpFixturesRoot(): string { + const dir = join(tmpdir(), `cat11-test-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`); + mkdirSync(dir, { recursive: true }); + return dir; +} + +// ─── Pure math helpers ──────────────────────────────────────────────── + +describe('charSimilarity', () => { + test('identical strings → 1', () => { + expect(charSimilarity('hello world', 'hello world')).toBe(1); + }); + + test('empty vs non-empty → 0', () => { + expect(charSimilarity('', 'hello')).toBe(0); + expect(charSimilarity('hello', '')).toBe(0); + }); + + test('both empty → 1', () => { + expect(charSimilarity('', '')).toBe(1); + }); + + test('one-char substitution', () => { + // 'cat' vs 'bat' = 1 edit over max 3 chars = 1 - 1/3 = 0.667 + expect(charSimilarity('cat', 'bat')).toBeCloseTo(2 / 3, 6); + }); +}); + +describe('wordRecall', () => { + test('all canonical words present → 1', () => { + expect(wordRecall('the quick brown fox', 'the lazy fox jumped over the quick brown dog')).toBe(1); + }); + + test('none present → 0', () => { + expect(wordRecall('one two three', 'alpha beta gamma')).toBe(0); + }); + + test('empty canonical → 1 (trivially satisfied)', () => { + expect(wordRecall('', 'anything')).toBe(1); + }); + + test('multiset semantics: three "the" vs one "the" → 1/3', () => { + expect(wordRecall('the the the', 'the cat')).toBeCloseTo(1 / 3, 6); + }); + + test('case-insensitive + strips punctuation', () => { + expect(wordRecall('Hello, world!', 'hello world')).toBe(1); + }); +}); + +describe('wer', () => { + test('perfect transcription → 0', () => { + expect(wer('hello world', 'hello world')).toBe(0); + }); + + test('one word wrong → 0.5 on 2-word reference', () => { + expect(wer('hello world', 'hello there')).toBe(0.5); + }); + + test('empty reference vs empty hypothesis → 0', () => { + expect(wer('', '')).toBe(0); + }); + + test('empty reference + non-empty hypothesis → 1', () => { + expect(wer('', 'anything')).toBe(1); + }); + + test('completely wrong → 1', () => { + expect(wer('one two three', 'alpha beta gamma')).toBe(1); + }); +}); + +describe('htmlToText', () => { + test('strips tags', () => { + expect(htmlToText('

hello world

')).toBe('hello world'); + }); + + test('drops script + style contents entirely', () => { + const html = 'visible'; + const out = htmlToText(html); + expect(out).toContain('visible'); + expect(out).not.toContain('alert'); + expect(out).not.toContain('body{}'); + }); + + test('decodes common entities', () => { + expect(htmlToText('<tag> & "quote"')).toContain(' & "quote"'); + }); + + test('collapses whitespace', () => { + expect(htmlToText('

a\n\nb \t c

')).toBe('a b c'); + }); +}); + +// ─── runHtmlModality (real filesystem + real manifest) ──────────────── + +describe('runHtmlModality', () => { + let root: string; + beforeEach(() => { + root = tmpFixturesRoot(); + }); + afterEach(() => { + rmSync(root, { recursive: true, force: true }); + }); + + test('skips gracefully when manifest missing', async () => { + const result = await runHtmlModality(root); + expect(result.skipped).toBe(true); + expect(result.skip_reason).toContain('manifest'); + }); + + test('scores real HTML fixture against canonical', async () => { + writeFileSync(join(root, 'page.html'), '

hello world

'); + writeFileSync(join(root, 'page.txt'), 'hello world'); + writeFileSync( + join(root, 'fixtures.json'), + JSON.stringify({ + version: 1, + license: 'CC-BY-SA', + items: [ + { + name: 'page', + path: 'page.html', + canonical_path: 'page.txt', + sha256: 'unused-in-test', + }, + ], + }), + ); + const result = await runHtmlModality(root); + expect(result.skipped).toBe(false); + expect(result.items).toBe(1); + expect(result.items_attempted).toBe(1); + expect(result.mean_metric).toBe(1); // canonical "hello world" recovered exactly + expect(result.per_item[0].name).toBe('page'); + }); + + test('missing fixture file produces an error entry (not a crash)', async () => { + writeFileSync( + join(root, 'fixtures.json'), + JSON.stringify({ + version: 1, + license: 'CC-BY-SA', + items: [{ name: 'ghost', path: 'ghost.html', canonical_path: 'ghost.txt', sha256: 'x' }], + }), + ); + const result = await runHtmlModality(root); + expect(result.skipped).toBe(false); + expect(result.per_item[0].error).toContain('missing'); + expect(result.items_attempted).toBe(0); + }); +}); + +// ─── runAudioModality ───────────────────────────────────────────────── + +describe('runAudioModality', () => { + let root: string; + const originalGroq = process.env.GROQ_API_KEY; + const originalOpenAI = process.env.OPENAI_API_KEY; + + beforeEach(() => { + root = tmpFixturesRoot(); + delete process.env.GROQ_API_KEY; + delete process.env.OPENAI_API_KEY; + }); + afterEach(() => { + rmSync(root, { recursive: true, force: true }); + if (originalGroq !== undefined) process.env.GROQ_API_KEY = originalGroq; + if (originalOpenAI !== undefined) process.env.OPENAI_API_KEY = originalOpenAI; + }); + + test('skips gracefully when manifest missing', async () => { + const result = await runAudioModality(root); + expect(result.skipped).toBe(true); + expect(result.skip_reason).toContain('manifest'); + }); + + test('skips gracefully when manifest present but no API key + no stub', async () => { + writeFileSync( + join(root, 'fixtures.json'), + JSON.stringify({ + version: 1, + license: 'CC0', + items: [{ name: 'clip', path: 'clip.mp3', canonical_path: 'clip.txt', sha256: 'x' }], + }), + ); + const result = await runAudioModality(root); + expect(result.skipped).toBe(true); + expect(result.skip_reason).toContain('GROQ_API_KEY'); + }); + + test('uses injected transcriber stub and computes WER', async () => { + writeFileSync(join(root, 'clip.mp3'), 'fake audio bytes'); + writeFileSync(join(root, 'clip.txt'), 'hello world this is a test'); + writeFileSync( + join(root, 'fixtures.json'), + JSON.stringify({ + version: 1, + license: 'CC0', + items: [{ name: 'clip', path: 'clip.mp3', canonical_path: 'clip.txt', sha256: 'x' }], + }), + ); + const stubTranscribe = async () => ({ text: 'hello world this is a test', provider: 'stub' }); + const result = await runAudioModality(root, { transcribe: stubTranscribe }); + expect(result.skipped).toBe(false); + expect(result.mean_metric).toBe(0); // perfect transcription → WER 0 + }); + + test('stub that returns wrong transcription produces non-zero WER', async () => { + writeFileSync(join(root, 'clip.mp3'), 'bytes'); + writeFileSync(join(root, 'clip.txt'), 'one two three four'); + writeFileSync( + join(root, 'fixtures.json'), + JSON.stringify({ + version: 1, + license: 'CC0', + items: [{ name: 'clip', path: 'clip.mp3', canonical_path: 'clip.txt', sha256: 'x' }], + }), + ); + const stub = async () => ({ text: 'one two three five', provider: 'stub' }); + const result = await runAudioModality(root, { transcribe: stub }); + expect(result.skipped).toBe(false); + expect(result.mean_metric).toBe(0.25); // 1 word wrong / 4 words = 0.25 + }); +}); + +// ─── runPdfModality ─────────────────────────────────────────────────── + +describe('runPdfModality', () => { + let root: string; + beforeEach(() => { root = tmpFixturesRoot(); }); + afterEach(() => { rmSync(root, { recursive: true, force: true }); }); + + test('skips gracefully when manifest missing', async () => { + const result = await runPdfModality(root); + expect(result.skipped).toBe(true); + expect(result.skip_reason).toContain('manifest'); + }); + + test('missing PDF file produces error entry (not crash)', async () => { + writeFileSync( + join(root, 'fixtures.json'), + JSON.stringify({ + version: 1, + license: 'CC-BY', + items: [{ name: 'paper', path: 'ghost.pdf', canonical_path: 'ghost.txt', sha256: 'x' }], + }), + ); + const result = await runPdfModality(root); + expect(result.skipped).toBe(false); + expect(result.per_item[0].error).toContain('missing'); + }); +}); + +// ─── runCat11 full dispatch ─────────────────────────────────────────── + +describe('runCat11', () => { + let root: string; + beforeEach(() => { root = tmpFixturesRoot(); }); + afterEach(() => { rmSync(root, { recursive: true, force: true }); }); + + test('returns skipped verdict when no modality has fixtures', async () => { + const report = await runCat11({ fixturesRoot: root }); + expect(report.verdict).toBe('skipped'); + expect(report.results.pdf.skipped).toBe(true); + expect(report.results.html.skipped).toBe(true); + expect(report.results.audio.skipped).toBe(true); + }); + + test('returns baseline_only when at least one modality ran', async () => { + // Set up an HTML fixture so at least one modality runs + mkdirSync(join(root, 'html'), { recursive: true }); + writeFileSync(join(root, 'html/p.html'), 'hello'); + writeFileSync(join(root, 'html/p.txt'), 'hello'); + writeFileSync( + join(root, 'html/fixtures.json'), + JSON.stringify({ + version: 1, + license: 'CC-BY-SA', + items: [{ name: 'p', path: 'p.html', canonical_path: 'p.txt', sha256: 'x' }], + }), + ); + const report = await runCat11({ fixturesRoot: root }); + expect(report.verdict).toBe('baseline_only'); + expect(report.results.html.skipped).toBe(false); + }); +}); diff --git a/test/eval/cat6.test.ts b/test/eval/cat6.test.ts new file mode 100644 index 000000000..bc370b26e --- /dev/null +++ b/test/eval/cat6.test.ts @@ -0,0 +1,202 @@ +/** + * cat6-prose-scale.ts tests — Day 6 of BrainBench v1 Complete. + * + * Uses a tiny synthetic corpus (5 pages) so tests run in <200ms, not the + * full 240-page world-v1 which is expensive to traverse per-test. + * + * Covers: + * - generateVariants produces deterministic variantIds under fixed seed + * - Variants cover all 6 injection kinds + * - scoreVariant computes matched/missed/false_positives correctly + * - aggregate produces all per-kind metrics + overall numbers + * - Verdict is always 'baseline_only' in v1 (no gating threshold) + */ + +import { describe, test, expect } from 'bun:test'; +import { + generateVariants, + aggregate, + scoreVariant, + makeCorpusResolver, + type BasePage, +} from '../../eval/runner/cat6-prose-scale.ts'; +import { ALL_INJECTION_KINDS } from '../../eval/runner/adversarial-injections.ts'; + +const TINY_CORPUS: BasePage[] = [ + { + slug: 'people/amara', + type: 'person', + title: 'Amara Okafor', + content: 'Amara is a partner at [Halfway](companies/halfway).', + }, + { + slug: 'people/jordan', + type: 'person', + title: 'Jordan Park', + content: 'Jordan founded [NovaMind](companies/novamind) in 2023.', + }, + { + slug: 'people/mina', + type: 'person', + title: 'Mina Kapoor', + content: 'Mina runs [Threshold](companies/threshold).', + }, + { + slug: 'people/sarah', + type: 'person', + title: 'Sarah Chen', + content: 'Sarah advises several seed-stage founders.', + }, + { + slug: 'companies/halfway', + type: 'company', + title: 'Halfway Capital', + content: 'VC firm focused on climate + AI infrastructure.', + }, + { + slug: 'companies/novamind', + type: 'company', + title: 'NovaMind', + content: 'AI infrastructure startup.', + }, + { + slug: 'companies/threshold', + type: 'company', + title: 'Threshold', + content: 'Venture firm.', + }, +]; + +// ─── Variant generation ─────────────────────────────────────────────── + +describe('generateVariants', () => { + test('produces exactly perKind × 6 variants', () => { + const variants = generateVariants(TINY_CORPUS, { perKind: 3 }); + expect(variants.length).toBe(3 * ALL_INJECTION_KINDS.length); + }); + + test('variants are distributed across all kinds', () => { + const variants = generateVariants(TINY_CORPUS, { perKind: 2 }); + const kinds = new Set(variants.map(v => v.kind)); + expect(kinds.size).toBe(ALL_INJECTION_KINDS.length); + }); + + test('deterministic under fixed seed', () => { + const a = generateVariants(TINY_CORPUS, { perKind: 2, baseSeed: 42 }); + const b = generateVariants(TINY_CORPUS, { perKind: 2, baseSeed: 42 }); + expect(a.length).toBe(b.length); + for (let i = 0; i < a.length; i++) { + expect(a[i].variantId).toBe(b[i].variantId); + expect(a[i].content).toBe(b[i].content); + } + }); + + test('different seeds produce different content', () => { + const a = generateVariants(TINY_CORPUS, { perKind: 2, baseSeed: 42 }); + const b = generateVariants(TINY_CORPUS, { perKind: 2, baseSeed: 99 }); + // code_fence_leak's fake slug depends on seed; content will differ. + const aCodeFences = a.filter(v => v.kind === 'code_fence_leak'); + const bCodeFences = b.filter(v => v.kind === 'code_fence_leak'); + expect(aCodeFences[0].content).not.toBe(bCodeFences[0].content); + }); + + test('variantId follows "-v-" pattern', () => { + const variants = generateVariants(TINY_CORPUS, { perKind: 1 }); + for (const v of variants) { + expect(v.variantId).toMatch(/^.+-v\d+-[a-z_]+$/); + expect(v.variantId.endsWith(`-${v.kind}`)).toBe(true); + } + }); +}); + +// ─── scoreVariant ───────────────────────────────────────────────────── + +describe('scoreVariant', () => { + test('returns VariantResult with extracted, missed, false_positives, matched', async () => { + const resolver = makeCorpusResolver(TINY_CORPUS); + const variants = generateVariants(TINY_CORPUS, { perKind: 1 }); + const result = await scoreVariant(variants[0], resolver); + expect(result.variantId).toBe(variants[0].variantId); + expect(Array.isArray(result.extracted)).toBe(true); + expect(Array.isArray(result.false_positives)).toBe(true); + expect(Array.isArray(result.missed)).toBe(true); + expect(typeof result.matched).toBe('number'); + }); + + test('counts a must_extract slug as matched when extractor produces it', async () => { + // Multi-entity sentences pack ≥4 real refs and the resolver accepts all known slugs. + // The extractor should produce those links. + const resolver = makeCorpusResolver(TINY_CORPUS); + const variants = generateVariants(TINY_CORPUS, { perKind: 5 }); + const multi = variants.find(v => v.kind === 'multi_entity_sentence' && v.goldDelta.must_extract.length >= 4); + if (!multi) { + // Skip gracefully if TINY_CORPUS is too small + return; + } + const result = await scoreVariant(multi, resolver); + expect(result.matched).toBeGreaterThan(0); + }); +}); + +// ─── aggregate ──────────────────────────────────────────────────────── + +describe('aggregate', () => { + test('emits per_kind for every kind present', async () => { + const resolver = makeCorpusResolver(TINY_CORPUS); + const variants = generateVariants(TINY_CORPUS, { perKind: 2 }); + const results = await Promise.all(variants.map(v => scoreVariant(v, resolver))); + const report = aggregate(variants, results); + expect(report.per_kind.length).toBe(ALL_INJECTION_KINDS.length); + expect(report.variants).toBe(variants.length); + }); + + test('overall metrics are computed (precision/recall/f1 in 0-1 range)', async () => { + const resolver = makeCorpusResolver(TINY_CORPUS); + const variants = generateVariants(TINY_CORPUS, { perKind: 2 }); + const results = await Promise.all(variants.map(v => scoreVariant(v, resolver))); + const report = aggregate(variants, results); + expect(report.overall.link_precision).toBeGreaterThanOrEqual(0); + expect(report.overall.link_precision).toBeLessThanOrEqual(1); + expect(report.overall.link_recall).toBeGreaterThanOrEqual(0); + expect(report.overall.link_recall).toBeLessThanOrEqual(1); + expect(report.overall.link_f1).toBeGreaterThanOrEqual(0); + expect(report.overall.link_f1).toBeLessThanOrEqual(1); + }); + + test('verdict is always baseline_only in v1', async () => { + const resolver = makeCorpusResolver(TINY_CORPUS); + const variants = generateVariants(TINY_CORPUS, { perKind: 1 }); + const results = await Promise.all(variants.map(v => scoreVariant(v, resolver))); + const report = aggregate(variants, results); + expect(report.verdict).toBe('baseline_only'); + }); + + test('mean_links_per_page matches totalExtracted / variants', async () => { + const resolver = makeCorpusResolver(TINY_CORPUS); + const variants = generateVariants(TINY_CORPUS, { perKind: 1 }); + const results = await Promise.all(variants.map(v => scoreVariant(v, resolver))); + const report = aggregate(variants, results); + const totalExtracted = results.reduce((sum, r) => sum + r.extracted.length, 0); + expect(report.overall.mean_links_per_page).toBeCloseTo(totalExtracted / variants.length, 6); + }); +}); + +// ─── Corpus resolver ────────────────────────────────────────────────── + +describe('makeCorpusResolver', () => { + test('returns slug unchanged when slug is known', async () => { + const resolver = makeCorpusResolver(TINY_CORPUS); + expect(await resolver.resolve('people/amara')).toBe('people/amara'); + }); + + test('returns null when slug is unknown', async () => { + const resolver = makeCorpusResolver(TINY_CORPUS); + expect(await resolver.resolve('people/ghost')).toBeNull(); + }); + + test('does not resolve bare names in v1', async () => { + const resolver = makeCorpusResolver(TINY_CORPUS); + // Cat 6 scoring semantics: bare-name resolution is out of scope. + expect(await resolver.resolve('Amara Okafor')).toBeNull(); + }); +});