mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-31 04:07:52 +00:00
feat(eval): Day 6 — adversarial-injections + Cat 6 prose-scale + Cat 11 multi-modal
Three modules that together cover BrainBench v1 Cat 6 (prose-scale
extraction fidelity) and Cat 11 (multi-modal ingest fidelity).
**eval/runner/adversarial-injections.ts** — 6 deterministic content
transforms shared by Cat 10 (adversarial.ts, 22 hand-crafted cases) and
Cat 6 (prose-scale variants). Each injection produces a modified content
string + a structured GoldDelta describing what the extractor MUST and
MUST NOT produce. Kinds:
- code_fence_leak — fake [X](people/fake) inside ``` fence, must NOT extract
- inline_code_slug — `people/fake` in backticks, must NOT extract
- substring_collision — "SamAI" near real `people/sam`, exactly one link
- ambiguous_role — "works with" vs "works at", downgrade type to mentions
- prose_only_mention — strip markdown link syntax, bare name → mentions only
- multi_entity_sentence — pack 4+ entities into one clause, extract all
Mulberry32 PRNG keeps variant generation deterministic under fixed seed.
Codex flagged the original plan's wording ("extract injection engine from
adversarial.ts") as overstated — adversarial.ts is a static case list,
not a reusable engine. This module is NEW code.
**eval/runner/cat6-prose-scale.ts** — Runner. Loads world-v1, applies all
6 injection kinds to sampled base pages (default 50 variants per kind ×
6 kinds = 300 variants), runs extractPageLinks on each, compares to gold
delta. Emits per-kind + overall metrics (precision, recall, F1,
code_fence_leak_rate, substring_fp_rate, pages_with_links_coverage,
mean_links_per_page). **v1 verdict is always "baseline_only"** — no
gating threshold per codex fix #9 (current extractor residuals make
>0.80 unreachable; v1 records a baseline, regression guard triggers on
drop below it).
**eval/runner/cat11-multimodal.ts** — PDF + HTML + audio runners.
Fixtures load from eval/data/multimodal/<modality>/fixtures.json
manifests; each modality skips gracefully when manifest missing or
(audio) when neither GROQ_API_KEY nor OPENAI_API_KEY is set. Metrics:
- PDF: char-level similarity via Levenshtein + optional entity_recall
- HTML: word-recall over normalized tokens (multiset semantics)
- Audio: WER (word error rate) via Levenshtein on word sequences
Fixtures are NOT committed; a future eval:fetch-multimodal script will
download them hash-verified from public sources (arXiv CC-licensed
papers, Wikipedia CC-BY-SA, Common Voice CC0).
Injectable audio transcriber (`opts.transcribe`) means tests don't need
GROQ/OpenAI keys — stubbed transcriptions exercise the WER math path
directly.
**Tests (60 new):** adversarial-injections (19) — per-kind assertions +
dispatcher coverage + slug regex conformance; cat6 (12) — variant
determinism, scoreVariant shape, aggregate per-kind + overall metrics,
corpus resolver slug rules; cat11 (29) — charSimilarity / wordRecall /
wer math, htmlToText strips scripts + decodes entities, HTML modality
with real fixtures, audio modality gracefully skips without key + uses
stub transcriber correctly.
All 60 tests pass in 48ms + 41ms.
Total eval suite now: 227 pass, 0 fail.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
co-authored by
Claude Opus 4.6
Claude Opus 4.6
parent
05aec3feb8
commit
839258dc42
@@ -0,0 +1,316 @@
|
||||
/**
|
||||
* Adversarial injection kinds shared by Cat 10 (adversarial.ts — 22 hand-
|
||||
* crafted edge cases) and Cat 6 (cat6-prose-scale.ts — 500+ prose-page
|
||||
* variants). Each kind is a deterministic transform from a base page
|
||||
* content string to a modified content string + a gold-delta description
|
||||
* of what the extractor's output should look like.
|
||||
*
|
||||
* The module lives alongside adversarial.ts but is NEW code — adversarial.ts
|
||||
* itself is a static case list, not a reusable transform engine. That's why
|
||||
* the original plan said "extract the engine from adversarial.ts" but codex
|
||||
* caught the overstatement: there's nothing to extract, we're building
|
||||
* the engine fresh here.
|
||||
*
|
||||
* Each injection has a `kind` tag that flows into `eval/data/world-v1-
|
||||
* adversarial/_adversarial/{slug}.json` metadata so the Cat 6 scorer can
|
||||
* compute per-kind precision / recall / false-positive rate.
|
||||
*/
|
||||
|
||||
export type InjectionKind =
|
||||
| 'code_fence_leak' // fake [X](people/fake-slug) inside ``` fences — must NOT extract
|
||||
| 'inline_code_slug' // `people/x` in backticks — must NOT extract
|
||||
| 'substring_collision' // "SamAI" near real person "sam" — exactly one link
|
||||
| 'ambiguous_role' // "works with" vs "works at" — downgrade to `mentions`
|
||||
| 'prose_only_mention' // strip the [X](people/slug) syntax; bare name remains — `mentions` only
|
||||
| 'multi_entity_sentence';// 4+ entities in one clause — all N links extracted
|
||||
|
||||
export interface EntityRef {
|
||||
/** Slug matching the one-slash validator regex. */
|
||||
slug: string;
|
||||
/** Display name, used when building markdown links. */
|
||||
name: string;
|
||||
}
|
||||
|
||||
export interface InjectionInput {
|
||||
/** The base content to inject into. Usually compiled_truth from a world-v1 page. */
|
||||
content: string;
|
||||
/** Seed for deterministic variant selection. */
|
||||
seed: number;
|
||||
/** Available entity references for this corpus (sampled to build transforms). */
|
||||
refs: EntityRef[];
|
||||
/** Specific entities to use (overrides `refs` sampling; useful for tests). */
|
||||
forcedRefs?: EntityRef[];
|
||||
}
|
||||
|
||||
export interface InjectionResult {
|
||||
/** Transformed content with the injection applied. */
|
||||
content: string;
|
||||
/** Structured description of what the extractor should and should NOT produce. */
|
||||
goldDelta: GoldDelta;
|
||||
}
|
||||
|
||||
export interface GoldDelta {
|
||||
/**
|
||||
* Links that MUST NOT appear in extraction output. Used by Cat 6's
|
||||
* `false_positive_rate` metric. For code_fence_leak / inline_code_slug
|
||||
* this lists the fake slugs inside fences; for substring_collision the
|
||||
* near-miss match; etc.
|
||||
*/
|
||||
must_not_extract: Array<{ slug: string; reason: string }>;
|
||||
/**
|
||||
* Links that MUST appear in extraction output (exact match). Used by
|
||||
* Cat 6 recall + per-kind recall. Empty when the injection is purely
|
||||
* negative (e.g., isolated code_fence_leak test).
|
||||
*/
|
||||
must_extract: Array<{ slug: string; type: string; reason: string }>;
|
||||
/** Human-readable note for the scorer's error reporting. */
|
||||
note: string;
|
||||
}
|
||||
|
||||
// ─── Seeded PRNG (Mulberry32, matches amara-life.ts) ──────────────────
|
||||
|
||||
function createRng(seed: number): () => number {
|
||||
let state = seed >>> 0;
|
||||
return () => {
|
||||
state = (state + 0x6D2B79F5) | 0;
|
||||
let t = Math.imul(state ^ (state >>> 15), 1 | state);
|
||||
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
|
||||
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
||||
};
|
||||
}
|
||||
|
||||
function pick<T>(rng: () => number, xs: T[]): T {
|
||||
if (xs.length === 0) throw new Error('pick: cannot pick from empty array');
|
||||
return xs[Math.floor(rng() * xs.length)];
|
||||
}
|
||||
|
||||
// ─── Individual injections ────────────────────────────────────────────
|
||||
|
||||
export function injectCodeFenceLeak(input: InjectionInput): InjectionResult {
|
||||
const rng = createRng(input.seed);
|
||||
const realTarget = pick(rng, input.forcedRefs ?? input.refs);
|
||||
const fakeSlug = `people/fake-${input.seed % 10000}`;
|
||||
|
||||
const fenceBlock = [
|
||||
'```',
|
||||
`const example = "${fakeSlug}";`,
|
||||
`// See also [FakeName](${fakeSlug}) — should NOT extract from inside a fence`,
|
||||
'```',
|
||||
].join('\n');
|
||||
|
||||
// Real mention (OUTSIDE the fence) so the baseline link is still there.
|
||||
const realMention = `See also [${realTarget.name}](${realTarget.slug}) for more.`;
|
||||
|
||||
const content = `${input.content}\n\n${fenceBlock}\n\n${realMention}`;
|
||||
|
||||
return {
|
||||
content,
|
||||
goldDelta: {
|
||||
must_not_extract: [
|
||||
{
|
||||
slug: fakeSlug,
|
||||
reason: 'code_fence_leak: slug appears inside triple-backtick fence',
|
||||
},
|
||||
],
|
||||
must_extract: [
|
||||
{
|
||||
slug: realTarget.slug,
|
||||
type: 'mentions',
|
||||
reason: 'code_fence_leak: real mention outside the fence should still extract',
|
||||
},
|
||||
],
|
||||
note: `Injected fake slug ${fakeSlug} inside code fence + real mention ${realTarget.slug} outside.`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function injectInlineCodeSlug(input: InjectionInput): InjectionResult {
|
||||
const rng = createRng(input.seed);
|
||||
const realTarget = pick(rng, input.forcedRefs ?? input.refs);
|
||||
const fakeSlug = `people/inline-fake-${input.seed % 10000}`;
|
||||
|
||||
const content = `${input.content}\n\nUse the \`${fakeSlug}\` notation in code. Real ref: [${realTarget.name}](${realTarget.slug}).`;
|
||||
|
||||
return {
|
||||
content,
|
||||
goldDelta: {
|
||||
must_not_extract: [
|
||||
{
|
||||
slug: fakeSlug,
|
||||
reason: 'inline_code_slug: slug wrapped in single-backtick inline code',
|
||||
},
|
||||
],
|
||||
must_extract: [
|
||||
{ slug: realTarget.slug, type: 'mentions', reason: 'inline_code_slug: real mention outside inline code' },
|
||||
],
|
||||
note: `Injected fake slug ${fakeSlug} in inline code + real mention ${realTarget.slug}.`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function injectSubstringCollision(input: InjectionInput): InjectionResult {
|
||||
const rng = createRng(input.seed);
|
||||
const realTarget = pick(rng, input.forcedRefs ?? input.refs);
|
||||
// Create a substring collision: if realTarget.name is "Sam", the collision
|
||||
// is "SamAI". The extractor should match ONLY the [Sam](slug) reference,
|
||||
// NOT "SamAI" appearing as prose.
|
||||
const baseName = realTarget.name.split(' ')[0];
|
||||
const collisionWord = `${baseName}AI`;
|
||||
|
||||
const content = `${input.content}\n\nThe ${collisionWord} initiative (unrelated) was launched last quarter. In parallel, [${realTarget.name}](${realTarget.slug}) continued their work on climate tooling.`;
|
||||
|
||||
return {
|
||||
content,
|
||||
goldDelta: {
|
||||
must_not_extract: [
|
||||
{
|
||||
slug: `people/${baseName.toLowerCase()}ai`,
|
||||
reason: `substring_collision: "${collisionWord}" must not be auto-linked to a people/ slug`,
|
||||
},
|
||||
],
|
||||
must_extract: [
|
||||
{ slug: realTarget.slug, type: 'mentions', reason: 'substring_collision: real markdown link should extract' },
|
||||
],
|
||||
note: `Injected substring collision "${collisionWord}" near real mention ${realTarget.slug}.`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function injectAmbiguousRole(input: InjectionInput): InjectionResult {
|
||||
const rng = createRng(input.seed);
|
||||
const realTarget = pick(rng, input.forcedRefs ?? input.refs);
|
||||
// Replace "works at" → "works with" if present, else append a "works with"
|
||||
// sentence. "works with" should downgrade to `mentions`, not `works_at`.
|
||||
const hasWorksAt = /works at/i.test(input.content);
|
||||
const content = hasWorksAt
|
||||
? input.content.replace(/works at/gi, 'works with')
|
||||
: `${input.content}\n\nShe regularly works with [${realTarget.name}](${realTarget.slug}) on quarterly reviews.`;
|
||||
|
||||
return {
|
||||
content,
|
||||
goldDelta: {
|
||||
must_not_extract: [], // The slug IS extracted; we just want type = mentions, not works_at
|
||||
must_extract: [
|
||||
{
|
||||
slug: realTarget.slug,
|
||||
type: 'mentions',
|
||||
reason: 'ambiguous_role: "works with" is loose enough that type must downgrade from works_at to mentions',
|
||||
},
|
||||
],
|
||||
note: `Injected "works with" phrasing for ${realTarget.slug} (must not upgrade to works_at).`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function injectProseOnlyMention(input: InjectionInput): InjectionResult {
|
||||
const rng = createRng(input.seed);
|
||||
const realTarget = pick(rng, input.forcedRefs ?? input.refs);
|
||||
// Strip markdown link syntax around the target's name — leave the bare name
|
||||
// in prose. Extractor should only produce a `mentions` link (not `founded`
|
||||
// or any typed relation), because the evidence is just prose co-occurrence.
|
||||
const nameRe = new RegExp(
|
||||
`\\[${escapeRegex(realTarget.name)}\\]\\(${escapeRegex(realTarget.slug)}\\)`,
|
||||
'g',
|
||||
);
|
||||
const stripped = input.content.replace(nameRe, realTarget.name);
|
||||
const content =
|
||||
stripped.length === input.content.length
|
||||
? `${input.content}\n\n${realTarget.name} has been a familiar figure in the industry for years.`
|
||||
: stripped;
|
||||
|
||||
return {
|
||||
content,
|
||||
goldDelta: {
|
||||
must_not_extract: [],
|
||||
must_extract: [
|
||||
{
|
||||
slug: realTarget.slug,
|
||||
type: 'mentions',
|
||||
reason: 'prose_only_mention: bare name in prose resolves to mentions, not typed relation',
|
||||
},
|
||||
],
|
||||
note: `Stripped markdown link around ${realTarget.name}, leaving bare prose mention only.`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function injectMultiEntitySentence(input: InjectionInput): InjectionResult {
|
||||
const rng = createRng(input.seed);
|
||||
const availableRefs = input.forcedRefs ?? input.refs;
|
||||
const n = Math.min(5, availableRefs.length);
|
||||
if (n < 4) {
|
||||
// Degenerate — not enough refs to build a multi-entity sentence.
|
||||
return {
|
||||
content: input.content,
|
||||
goldDelta: {
|
||||
must_not_extract: [],
|
||||
must_extract: [],
|
||||
note: 'multi_entity_sentence: skipped (fewer than 4 refs available)',
|
||||
},
|
||||
};
|
||||
}
|
||||
const picks: EntityRef[] = [];
|
||||
const seen = new Set<string>();
|
||||
// Deterministic pick of n unique refs
|
||||
let attempts = 0;
|
||||
while (picks.length < n && attempts < n * 10) {
|
||||
const ref = pick(rng, availableRefs);
|
||||
if (!seen.has(ref.slug)) {
|
||||
picks.push(ref);
|
||||
seen.add(ref.slug);
|
||||
}
|
||||
attempts++;
|
||||
}
|
||||
|
||||
const mdLinks = picks.map(r => `[${r.name}](${r.slug})`);
|
||||
const clause =
|
||||
`At the most recent Halfway partners meeting, ` +
|
||||
`${mdLinks.slice(0, -1).join(', ')}, and ${mdLinks[mdLinks.length - 1]} all discussed the quarterly outlook.`;
|
||||
|
||||
const content = `${input.content}\n\n${clause}`;
|
||||
|
||||
return {
|
||||
content,
|
||||
goldDelta: {
|
||||
must_not_extract: [],
|
||||
must_extract: picks.map(r => ({
|
||||
slug: r.slug,
|
||||
type: 'mentions',
|
||||
reason: `multi_entity_sentence: all ${n} entities in a packed clause should extract`,
|
||||
})),
|
||||
note: `Injected packed clause with ${n} entities: ${picks.map(r => r.slug).join(', ')}.`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Dispatcher ──────────────────────────────────────────────────────
|
||||
|
||||
export const ALL_INJECTION_KINDS: readonly InjectionKind[] = [
|
||||
'code_fence_leak',
|
||||
'inline_code_slug',
|
||||
'substring_collision',
|
||||
'ambiguous_role',
|
||||
'prose_only_mention',
|
||||
'multi_entity_sentence',
|
||||
] as const;
|
||||
|
||||
export function applyInjection(kind: InjectionKind, input: InjectionInput): InjectionResult {
|
||||
switch (kind) {
|
||||
case 'code_fence_leak':
|
||||
return injectCodeFenceLeak(input);
|
||||
case 'inline_code_slug':
|
||||
return injectInlineCodeSlug(input);
|
||||
case 'substring_collision':
|
||||
return injectSubstringCollision(input);
|
||||
case 'ambiguous_role':
|
||||
return injectAmbiguousRole(input);
|
||||
case 'prose_only_mention':
|
||||
return injectProseOnlyMention(input);
|
||||
case 'multi_entity_sentence':
|
||||
return injectMultiEntitySentence(input);
|
||||
}
|
||||
}
|
||||
|
||||
function escapeRegex(s: string): string {
|
||||
return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
}
|
||||
@@ -0,0 +1,451 @@
|
||||
/**
|
||||
* Cat 11 — Multi-modal ingestion fidelity.
|
||||
*
|
||||
* Measures text fidelity across three modalities:
|
||||
*
|
||||
* - **PDF** (10 arXiv papers, CC-licensed): char-level similarity vs
|
||||
* canonical .txt + entity recall. Threshold (informational): >0.95.
|
||||
* - **Audio** (5 Common Voice CC0 clips): WER vs canonical .txt.
|
||||
* Opt-in via `GROQ_API_KEY` or `OPENAI_API_KEY`. Threshold: <0.15.
|
||||
* - **HTML** (10 Wikipedia snapshots, CC-BY-SA): prose word-recall vs
|
||||
* canonical .txt. Threshold: >0.80.
|
||||
*
|
||||
* Image OCR + video deferred to Cat 11 v2. XLSX also deferred (use authored
|
||||
* markdown tables instead — codex flagged the xlsx parser dep as not worth
|
||||
* adding for v1).
|
||||
*
|
||||
* Fixtures are NOT committed to the repo. They download on demand via
|
||||
* `bun run eval:fetch-multimodal` using hash-verified manifests at
|
||||
* `eval/data/multimodal/<pdf|audio|html>/fixtures.json`. This keeps the
|
||||
* repo lean while preserving reproducibility.
|
||||
*/
|
||||
|
||||
import { existsSync, readFileSync } from 'fs';
|
||||
import { join, resolve } from 'path';
|
||||
import { extractPdfText } from './loaders/pdf.ts';
|
||||
|
||||
// ─── Manifest types ──────────────────────────────────────────────────
|
||||
|
||||
export interface FixtureManifest {
|
||||
version: 1;
|
||||
license: string;
|
||||
items: Array<{
|
||||
name: string;
|
||||
path: string;
|
||||
canonical_path: string;
|
||||
sha256: string;
|
||||
/** For entity-recall scoring (PDF). Optional list of expected entity names. */
|
||||
entities?: string[];
|
||||
}>;
|
||||
}
|
||||
|
||||
// ─── Per-modality results ────────────────────────────────────────────
|
||||
|
||||
export interface ModalityResult {
|
||||
modality: 'pdf' | 'audio' | 'html';
|
||||
items: number;
|
||||
items_attempted: number;
|
||||
/** Mean similarity / fidelity / recall for the modality. */
|
||||
mean_metric: number;
|
||||
/** Per-item breakdown. */
|
||||
per_item: Array<{
|
||||
name: string;
|
||||
metric: number;
|
||||
/** Optional — e.g., number of entities matched (PDF) or word-count (HTML). */
|
||||
detail?: Record<string, number>;
|
||||
error?: string;
|
||||
}>;
|
||||
/** True if the modality was skipped (e.g., missing fixtures or API key). */
|
||||
skipped: boolean;
|
||||
skip_reason?: string;
|
||||
}
|
||||
|
||||
export interface Cat11Report {
|
||||
schema_version: 1;
|
||||
ran_at: string;
|
||||
results: Record<'pdf' | 'audio' | 'html', ModalityResult>;
|
||||
verdict: 'pass' | 'fail' | 'baseline_only' | 'skipped';
|
||||
}
|
||||
|
||||
// ─── Char-level similarity (Levenshtein-based) ───────────────────────
|
||||
|
||||
/**
|
||||
* Char-level similarity using normalized Levenshtein edit distance.
|
||||
* Returns 0..1 where 1 = exact match. O(n*m) memory — fine for PDF/HTML
|
||||
* comparisons capped at ~100KB each.
|
||||
*/
|
||||
export function charSimilarity(a: string, b: string): number {
|
||||
if (a === b) return 1;
|
||||
if (a.length === 0 || b.length === 0) return 0;
|
||||
const n = a.length;
|
||||
const m = b.length;
|
||||
// Use typed arrays for speed on larger strings
|
||||
let prev = new Int32Array(m + 1);
|
||||
let curr = new Int32Array(m + 1);
|
||||
for (let j = 0; j <= m; j++) prev[j] = j;
|
||||
for (let i = 1; i <= n; i++) {
|
||||
curr[0] = i;
|
||||
for (let j = 1; j <= m; j++) {
|
||||
const cost = a.charCodeAt(i - 1) === b.charCodeAt(j - 1) ? 0 : 1;
|
||||
curr[j] = Math.min(
|
||||
prev[j] + 1, // deletion
|
||||
curr[j - 1] + 1, // insertion
|
||||
prev[j - 1] + cost, // substitution
|
||||
);
|
||||
}
|
||||
[prev, curr] = [curr, prev];
|
||||
}
|
||||
const distance = prev[m];
|
||||
const maxLen = Math.max(n, m);
|
||||
return 1 - distance / maxLen;
|
||||
}
|
||||
|
||||
// ─── Word-level recall (for HTML) ────────────────────────────────────
|
||||
|
||||
function normalizeWords(s: string): string[] {
|
||||
return s
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9\s]+/g, ' ')
|
||||
.split(/\s+/)
|
||||
.filter(w => w.length > 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Word recall: fraction of canonical words present in extracted text.
|
||||
* Multiset semantics: if canonical has "the the the" and extracted has "the",
|
||||
* recall is 1/3.
|
||||
*/
|
||||
export function wordRecall(canonical: string, extracted: string): number {
|
||||
const canonicalWords = normalizeWords(canonical);
|
||||
const extractedCounts = new Map<string, number>();
|
||||
for (const w of normalizeWords(extracted)) {
|
||||
extractedCounts.set(w, (extractedCounts.get(w) ?? 0) + 1);
|
||||
}
|
||||
if (canonicalWords.length === 0) return 1;
|
||||
let hits = 0;
|
||||
for (const w of canonicalWords) {
|
||||
const count = extractedCounts.get(w) ?? 0;
|
||||
if (count > 0) {
|
||||
hits++;
|
||||
extractedCounts.set(w, count - 1);
|
||||
}
|
||||
}
|
||||
return hits / canonicalWords.length;
|
||||
}
|
||||
|
||||
// ─── WER (word error rate) for audio ─────────────────────────────────
|
||||
|
||||
/**
|
||||
* Word Error Rate: Levenshtein distance at word level / reference word count.
|
||||
* Lower is better. 0.0 = perfect transcription. Typical whisper-v3 on clean
|
||||
* English ≈ 0.05-0.10.
|
||||
*/
|
||||
export function wer(reference: string, hypothesis: string): number {
|
||||
const ref = normalizeWords(reference);
|
||||
const hyp = normalizeWords(hypothesis);
|
||||
if (ref.length === 0) return hyp.length === 0 ? 0 : 1;
|
||||
|
||||
const n = ref.length;
|
||||
const m = hyp.length;
|
||||
let prev = new Int32Array(m + 1);
|
||||
let curr = new Int32Array(m + 1);
|
||||
for (let j = 0; j <= m; j++) prev[j] = j;
|
||||
for (let i = 1; i <= n; i++) {
|
||||
curr[0] = i;
|
||||
for (let j = 1; j <= m; j++) {
|
||||
const cost = ref[i - 1] === hyp[j - 1] ? 0 : 1;
|
||||
curr[j] = Math.min(prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + cost);
|
||||
}
|
||||
[prev, curr] = [curr, prev];
|
||||
}
|
||||
return prev[m] / n;
|
||||
}
|
||||
|
||||
// ─── PDF modality ─────────────────────────────────────────────────────
|
||||
|
||||
export async function runPdfModality(
|
||||
fixturesDir: string = resolve(process.cwd(), 'eval/data/multimodal/pdf'),
|
||||
): Promise<ModalityResult> {
|
||||
const manifestPath = join(fixturesDir, 'fixtures.json');
|
||||
if (!existsSync(manifestPath)) {
|
||||
return {
|
||||
modality: 'pdf',
|
||||
items: 0,
|
||||
items_attempted: 0,
|
||||
mean_metric: 0,
|
||||
per_item: [],
|
||||
skipped: true,
|
||||
skip_reason: `No manifest at ${manifestPath}. Run \`bun run eval:fetch-multimodal\` first.`,
|
||||
};
|
||||
}
|
||||
const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) as FixtureManifest;
|
||||
const perItem: ModalityResult['per_item'] = [];
|
||||
let metricSum = 0;
|
||||
let attempted = 0;
|
||||
|
||||
for (const item of manifest.items) {
|
||||
const pdfPath = join(fixturesDir, item.path);
|
||||
const canonicalPath = join(fixturesDir, item.canonical_path);
|
||||
if (!existsSync(pdfPath) || !existsSync(canonicalPath)) {
|
||||
perItem.push({
|
||||
name: item.name,
|
||||
metric: 0,
|
||||
error: `missing fixture file(s) at ${item.path} / ${item.canonical_path}`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const extracted = await extractPdfText(pdfPath);
|
||||
const canonical = readFileSync(canonicalPath, 'utf8');
|
||||
const similarity = charSimilarity(canonical.trim(), extracted.text.trim());
|
||||
|
||||
// Entity recall if the manifest declares expected entities
|
||||
let entityRecall: number | undefined;
|
||||
if (item.entities && item.entities.length > 0) {
|
||||
const hits = item.entities.filter(name =>
|
||||
extracted.text.toLowerCase().includes(name.toLowerCase()),
|
||||
).length;
|
||||
entityRecall = hits / item.entities.length;
|
||||
}
|
||||
|
||||
perItem.push({
|
||||
name: item.name,
|
||||
metric: similarity,
|
||||
detail: {
|
||||
num_pages: extracted.numPages,
|
||||
canonical_chars: canonical.length,
|
||||
extracted_chars: extracted.text.length,
|
||||
...(entityRecall !== undefined ? { entity_recall: entityRecall } : {}),
|
||||
},
|
||||
});
|
||||
metricSum += similarity;
|
||||
attempted++;
|
||||
} catch (err) {
|
||||
perItem.push({ name: item.name, metric: 0, error: String(err) });
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
modality: 'pdf',
|
||||
items: manifest.items.length,
|
||||
items_attempted: attempted,
|
||||
mean_metric: attempted === 0 ? 0 : metricSum / attempted,
|
||||
per_item: perItem,
|
||||
skipped: false,
|
||||
};
|
||||
}
|
||||
|
||||
// ─── HTML modality ────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Strip HTML tags + common noise, collapse whitespace. Not a full HTML
|
||||
* parser — intentionally simple to make the metric stable across HTML
|
||||
* variations.
|
||||
*/
|
||||
export function htmlToText(html: string): string {
|
||||
return html
|
||||
.replace(/<script[\s\S]*?<\/script>/gi, ' ')
|
||||
.replace(/<style[\s\S]*?<\/style>/gi, ' ')
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/ /gi, ' ')
|
||||
.replace(/&/gi, '&')
|
||||
.replace(/</gi, '<')
|
||||
.replace(/>/gi, '>')
|
||||
.replace(/"/gi, '"')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
export async function runHtmlModality(
|
||||
fixturesDir: string = resolve(process.cwd(), 'eval/data/multimodal/html'),
|
||||
): Promise<ModalityResult> {
|
||||
const manifestPath = join(fixturesDir, 'fixtures.json');
|
||||
if (!existsSync(manifestPath)) {
|
||||
return {
|
||||
modality: 'html',
|
||||
items: 0,
|
||||
items_attempted: 0,
|
||||
mean_metric: 0,
|
||||
per_item: [],
|
||||
skipped: true,
|
||||
skip_reason: `No manifest at ${manifestPath}.`,
|
||||
};
|
||||
}
|
||||
const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) as FixtureManifest;
|
||||
const perItem: ModalityResult['per_item'] = [];
|
||||
let metricSum = 0;
|
||||
let attempted = 0;
|
||||
|
||||
for (const item of manifest.items) {
|
||||
const htmlPath = join(fixturesDir, item.path);
|
||||
const canonicalPath = join(fixturesDir, item.canonical_path);
|
||||
if (!existsSync(htmlPath) || !existsSync(canonicalPath)) {
|
||||
perItem.push({
|
||||
name: item.name,
|
||||
metric: 0,
|
||||
error: `missing fixture file(s)`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const html = readFileSync(htmlPath, 'utf8');
|
||||
const canonical = readFileSync(canonicalPath, 'utf8');
|
||||
const extracted = htmlToText(html);
|
||||
const recall = wordRecall(canonical, extracted);
|
||||
perItem.push({
|
||||
name: item.name,
|
||||
metric: recall,
|
||||
detail: {
|
||||
canonical_chars: canonical.length,
|
||||
extracted_chars: extracted.length,
|
||||
},
|
||||
});
|
||||
metricSum += recall;
|
||||
attempted++;
|
||||
} catch (err) {
|
||||
perItem.push({ name: item.name, metric: 0, error: String(err) });
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
modality: 'html',
|
||||
items: manifest.items.length,
|
||||
items_attempted: attempted,
|
||||
mean_metric: attempted === 0 ? 0 : metricSum / attempted,
|
||||
per_item: perItem,
|
||||
skipped: false,
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Audio modality ──────────────────────────────────────────────────
|
||||
|
||||
export interface AudioRunOpts {
|
||||
/** Injectable transcriber for tests. Default uses src/core/transcription.ts. */
|
||||
transcribe?: (audioPath: string) => Promise<{ text: string; provider: string }>;
|
||||
}
|
||||
|
||||
export async function runAudioModality(
|
||||
fixturesDir: string = resolve(process.cwd(), 'eval/data/multimodal/audio'),
|
||||
opts: AudioRunOpts = {},
|
||||
): Promise<ModalityResult> {
|
||||
const manifestPath = join(fixturesDir, 'fixtures.json');
|
||||
if (!existsSync(manifestPath)) {
|
||||
return {
|
||||
modality: 'audio',
|
||||
items: 0,
|
||||
items_attempted: 0,
|
||||
mean_metric: 0,
|
||||
per_item: [],
|
||||
skipped: true,
|
||||
skip_reason: `No manifest at ${manifestPath}.`,
|
||||
};
|
||||
}
|
||||
|
||||
// Gate on API keys unless a test-injected transcriber is provided.
|
||||
const hasApiKey = !!process.env.GROQ_API_KEY || !!process.env.OPENAI_API_KEY;
|
||||
if (!opts.transcribe && !hasApiKey) {
|
||||
return {
|
||||
modality: 'audio',
|
||||
items: 0,
|
||||
items_attempted: 0,
|
||||
mean_metric: 0,
|
||||
per_item: [],
|
||||
skipped: true,
|
||||
skip_reason: 'Neither GROQ_API_KEY nor OPENAI_API_KEY is set; audio transcription requires one.',
|
||||
};
|
||||
}
|
||||
|
||||
const transcriber = opts.transcribe ?? (await loadDefaultTranscriber());
|
||||
|
||||
const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) as FixtureManifest;
|
||||
const perItem: ModalityResult['per_item'] = [];
|
||||
let metricSum = 0;
|
||||
let attempted = 0;
|
||||
|
||||
for (const item of manifest.items) {
|
||||
const audioPath = join(fixturesDir, item.path);
|
||||
const canonicalPath = join(fixturesDir, item.canonical_path);
|
||||
if (!existsSync(audioPath) || !existsSync(canonicalPath)) {
|
||||
perItem.push({ name: item.name, metric: 1, error: 'missing fixture file(s)' });
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const { text, provider } = await transcriber(audioPath);
|
||||
const canonical = readFileSync(canonicalPath, 'utf8');
|
||||
const errorRate = wer(canonical, text);
|
||||
perItem.push({
|
||||
name: item.name,
|
||||
metric: errorRate,
|
||||
detail: {
|
||||
canonical_words: normalizeWords(canonical).length,
|
||||
transcribed_words: normalizeWords(text).length,
|
||||
provider_length: provider.length,
|
||||
},
|
||||
});
|
||||
metricSum += errorRate;
|
||||
attempted++;
|
||||
} catch (err) {
|
||||
perItem.push({ name: item.name, metric: 1, error: String(err) });
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
modality: 'audio',
|
||||
items: manifest.items.length,
|
||||
items_attempted: attempted,
|
||||
mean_metric: attempted === 0 ? 0 : metricSum / attempted,
|
||||
per_item: perItem,
|
||||
skipped: false,
|
||||
};
|
||||
}
|
||||
|
||||
async function loadDefaultTranscriber(): Promise<NonNullable<AudioRunOpts['transcribe']>> {
|
||||
// Lazy import src/core/transcription.ts so this file doesn't drag that
|
||||
// import into module-load time. transcription.ts has env checks that
|
||||
// throw if GROQ/OPENAI keys are absent.
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const mod: any = await import('../../src/core/transcription.ts');
|
||||
const fn = mod.transcribe ?? mod.default;
|
||||
if (typeof fn !== 'function') {
|
||||
throw new Error('src/core/transcription.ts does not export a transcribe function');
|
||||
}
|
||||
return async (audioPath: string) => {
|
||||
const res = await fn(audioPath);
|
||||
return { text: res.text ?? '', provider: res.provider ?? 'unknown' };
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Runner entry ─────────────────────────────────────────────────────
|
||||
|
||||
export interface RunCat11Options {
|
||||
fixturesRoot?: string;
|
||||
/** Override the audio transcriber (test injection). */
|
||||
transcribeAudio?: AudioRunOpts['transcribe'];
|
||||
}
|
||||
|
||||
export async function runCat11(opts: RunCat11Options = {}): Promise<Cat11Report> {
|
||||
const root = opts.fixturesRoot ?? resolve(process.cwd(), 'eval/data/multimodal');
|
||||
const [pdf, html, audio] = await Promise.all([
|
||||
runPdfModality(join(root, 'pdf')),
|
||||
runHtmlModality(join(root, 'html')),
|
||||
runAudioModality(join(root, 'audio'), { transcribe: opts.transcribeAudio }),
|
||||
]);
|
||||
|
||||
const anyRan = !pdf.skipped || !html.skipped || !audio.skipped;
|
||||
|
||||
return {
|
||||
schema_version: 1,
|
||||
ran_at: new Date().toISOString(),
|
||||
results: { pdf, html, audio },
|
||||
verdict: anyRan ? 'baseline_only' : 'skipped',
|
||||
};
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
runCat11()
|
||||
.then(report => console.log(JSON.stringify(report, null, 2)))
|
||||
.catch(err => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,402 @@
|
||||
/**
|
||||
* Cat 6 — Auto-link Precision under Prose (at scale).
|
||||
*
|
||||
* Extends Cat 10 (adversarial.ts, 22 hand-crafted edge cases) to N prose-heavy
|
||||
* page variants generated by applying the 6 injections from
|
||||
* `adversarial-injections.ts` to world-v1 base pages.
|
||||
*
|
||||
* **v1 is baseline-only** (no gating threshold). Codex flagged that
|
||||
* `type_accuracy > 0.80` is unreachable given current residuals
|
||||
* (`works_at` 58%, `advises` 41% per TODOS.md). Cat 6 in v1 produces a
|
||||
* numeric baseline the scorecard records; regression guard is
|
||||
* "score must not drop below committed first baseline." Threshold gating
|
||||
* returns after the extractor uplift TODO lands.
|
||||
*
|
||||
* Metrics:
|
||||
* - link_precision, link_recall, link_f1 (overall)
|
||||
* - per-injection-kind: precision, recall, false_positive_rate
|
||||
* - code_fence_leak_rate = leaked_fence_links / total_fence_cases
|
||||
* - substring_fp_rate = collided_substring_links / total_collision_cases
|
||||
* - pages_with_links_coverage (fraction of variants producing ≥1 link)
|
||||
* - mean_links_per_page
|
||||
*/
|
||||
|
||||
import { readdirSync, readFileSync } from 'fs';
|
||||
import { join, resolve } from 'path';
|
||||
import {
|
||||
ALL_INJECTION_KINDS,
|
||||
applyInjection,
|
||||
type EntityRef,
|
||||
type GoldDelta,
|
||||
type InjectionKind,
|
||||
} from './adversarial-injections.ts';
|
||||
import { extractPageLinks, type SlugResolver } from '../../src/core/link-extraction.ts';
|
||||
import type { PageType } from '../../src/core/types.ts';
|
||||
|
||||
// ─── Types ────────────────────────────────────────────────────────────
|
||||
|
||||
export interface VariantCase {
|
||||
/** Unique id for this variant, e.g. "people/alice-v0-code_fence_leak". */
|
||||
variantId: string;
|
||||
/** Base slug the variant was derived from. */
|
||||
baseSlug: string;
|
||||
baseType: PageType;
|
||||
/** Injection applied to the base content. */
|
||||
kind: InjectionKind;
|
||||
/** Transformed content (what the extractor sees). */
|
||||
content: string;
|
||||
/** Gold delta from the injection. */
|
||||
goldDelta: GoldDelta;
|
||||
}
|
||||
|
||||
export interface ExtractedLink {
|
||||
targetSlug: string;
|
||||
linkType: string;
|
||||
}
|
||||
|
||||
export interface VariantResult {
|
||||
variantId: string;
|
||||
kind: InjectionKind;
|
||||
extracted: ExtractedLink[];
|
||||
/** Slugs that appeared in extraction output but were marked must_not_extract. */
|
||||
false_positives: Array<{ slug: string; reason: string }>;
|
||||
/** Slugs from must_extract that DID NOT appear in extraction output. */
|
||||
missed: Array<{ slug: string; type: string; reason: string }>;
|
||||
/** Slugs from must_extract that DID appear (regardless of type match). */
|
||||
matched: number;
|
||||
}
|
||||
|
||||
export interface PerKindReport {
|
||||
kind: InjectionKind;
|
||||
variants: number;
|
||||
total_must_extract: number;
|
||||
total_matched: number;
|
||||
total_false_positives: number;
|
||||
precision: number;
|
||||
recall: number;
|
||||
false_positive_rate: number;
|
||||
}
|
||||
|
||||
export interface Cat6Report {
|
||||
schema_version: 1;
|
||||
ran_at: string;
|
||||
variants: number;
|
||||
corpus_id: string;
|
||||
per_kind: PerKindReport[];
|
||||
overall: {
|
||||
link_precision: number;
|
||||
link_recall: number;
|
||||
link_f1: number;
|
||||
code_fence_leak_rate: number;
|
||||
substring_fp_rate: number;
|
||||
pages_with_links_coverage: number;
|
||||
mean_links_per_page: number;
|
||||
};
|
||||
verdict: 'baseline_only';
|
||||
}
|
||||
|
||||
// ─── Corpus loading ──────────────────────────────────────────────────
|
||||
|
||||
export interface BasePage {
|
||||
slug: string;
|
||||
type: PageType;
|
||||
title: string;
|
||||
content: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Minimal loader for `eval/data/world-v1` JSON shards. Each shard has shape
|
||||
* `{slug, type, title, compiled_truth, timeline, _facts}`. We concatenate
|
||||
* compiled_truth + timeline as the extractor sees them.
|
||||
*/
|
||||
export function loadWorldV1(
|
||||
corpusDir: string = resolve(process.cwd(), 'eval/data/world-v1'),
|
||||
): BasePage[] {
|
||||
const files = readdirSync(corpusDir).filter(
|
||||
f => f.endsWith('.json') && !f.startsWith('_'),
|
||||
);
|
||||
const pages: BasePage[] = [];
|
||||
for (const f of files) {
|
||||
const raw = readFileSync(join(corpusDir, f), 'utf8');
|
||||
const shard = JSON.parse(raw) as {
|
||||
slug: string;
|
||||
type: PageType;
|
||||
title: string;
|
||||
compiled_truth: string;
|
||||
timeline?: string;
|
||||
};
|
||||
pages.push({
|
||||
slug: shard.slug,
|
||||
type: shard.type,
|
||||
title: shard.title,
|
||||
content: `${shard.compiled_truth}\n\n${shard.timeline ?? ''}`,
|
||||
});
|
||||
}
|
||||
return pages;
|
||||
}
|
||||
|
||||
// ─── Variant generation ──────────────────────────────────────────────
|
||||
|
||||
function pagesToRefs(pages: BasePage[]): EntityRef[] {
|
||||
return pages
|
||||
.filter(p => p.type === 'person' || p.type === 'company')
|
||||
.map(p => ({ slug: p.slug, name: p.title }));
|
||||
}
|
||||
|
||||
export interface VariantGenerationOpts {
|
||||
/** Max variants per injection kind. Default 50 → 50 × 6 = 300 variants total. */
|
||||
perKind?: number;
|
||||
/** Seed offset; each variant computes seed = baseSeed + variantIdx. Default 1000. */
|
||||
baseSeed?: number;
|
||||
/** Optional set of base-page types to include. Default ['person', 'company']. */
|
||||
includeTypes?: PageType[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate variant cases deterministically from a base corpus. Variants are
|
||||
* spread across all 6 injection kinds; seed determinism means re-running
|
||||
* against the same corpus produces identical variants.
|
||||
*/
|
||||
export function generateVariants(
|
||||
pages: BasePage[],
|
||||
opts: VariantGenerationOpts = {},
|
||||
): VariantCase[] {
|
||||
const perKind = opts.perKind ?? 50;
|
||||
const baseSeed = opts.baseSeed ?? 1000;
|
||||
const includeTypes = opts.includeTypes ?? (['person', 'company'] as PageType[]);
|
||||
const validPages = pages.filter(p => includeTypes.includes(p.type));
|
||||
if (validPages.length === 0) return [];
|
||||
|
||||
const refs = pagesToRefs(pages);
|
||||
|
||||
const variants: VariantCase[] = [];
|
||||
let variantIdx = 0;
|
||||
|
||||
for (const kind of ALL_INJECTION_KINDS) {
|
||||
for (let i = 0; i < perKind; i++) {
|
||||
// Deterministic base page pick: step through the corpus in order.
|
||||
const base = validPages[(variantIdx + i) % validPages.length];
|
||||
const seed = baseSeed + variantIdx;
|
||||
const res = applyInjection(kind, { content: base.content, seed, refs });
|
||||
variants.push({
|
||||
variantId: `${base.slug}-v${i}-${kind}`,
|
||||
baseSlug: base.slug,
|
||||
baseType: base.type,
|
||||
kind,
|
||||
content: res.content,
|
||||
goldDelta: res.goldDelta,
|
||||
});
|
||||
variantIdx++;
|
||||
}
|
||||
}
|
||||
|
||||
return variants;
|
||||
}
|
||||
|
||||
// ─── Extraction + scoring ─────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Minimal slug resolver: accepts only slugs present in the corpus. Returns
|
||||
* `slug` unchanged if known, `null` otherwise. This matches Cat 6 semantics
|
||||
* because the adversarial transforms never introduce NEW valid target slugs
|
||||
* — any "link" the extractor produces must resolve to something real.
|
||||
*/
|
||||
export function makeCorpusResolver(pages: BasePage[]): SlugResolver {
|
||||
const known = new Set(pages.map(p => p.slug));
|
||||
return {
|
||||
resolve: async (name: string) => {
|
||||
// Exact slug match — bare-name resolution is out of scope for Cat 6 scoring.
|
||||
if (known.has(name)) return name;
|
||||
return null;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function extractOne(
|
||||
variant: VariantCase,
|
||||
resolver: SlugResolver,
|
||||
): Promise<ExtractedLink[]> {
|
||||
const res = await extractPageLinks(
|
||||
variant.baseSlug,
|
||||
variant.content,
|
||||
{}, // frontmatter unused for Cat 6 prose tests
|
||||
variant.baseType,
|
||||
resolver,
|
||||
);
|
||||
return res.candidates
|
||||
.filter(c => c.targetSlug)
|
||||
.map(c => ({ targetSlug: c.targetSlug, linkType: c.linkType ?? 'mentions' }));
|
||||
}
|
||||
|
||||
export async function scoreVariant(
|
||||
variant: VariantCase,
|
||||
resolver: SlugResolver,
|
||||
): Promise<VariantResult> {
|
||||
const extracted = await extractOne(variant, resolver);
|
||||
const extractedSlugs = new Set(extracted.map(e => e.targetSlug));
|
||||
|
||||
const false_positives: Array<{ slug: string; reason: string }> = [];
|
||||
for (const must_not of variant.goldDelta.must_not_extract) {
|
||||
if (extractedSlugs.has(must_not.slug)) {
|
||||
false_positives.push(must_not);
|
||||
}
|
||||
}
|
||||
|
||||
const missed: Array<{ slug: string; type: string; reason: string }> = [];
|
||||
let matched = 0;
|
||||
for (const must of variant.goldDelta.must_extract) {
|
||||
if (extractedSlugs.has(must.slug)) {
|
||||
matched++;
|
||||
} else {
|
||||
missed.push(must);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
variantId: variant.variantId,
|
||||
kind: variant.kind,
|
||||
extracted,
|
||||
false_positives,
|
||||
missed,
|
||||
matched,
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Aggregation ──────────────────────────────────────────────────────
|
||||
|
||||
function precision(matched: number, extracted: number): number {
|
||||
return extracted === 0 ? 0 : matched / extracted;
|
||||
}
|
||||
|
||||
function recall(matched: number, expected: number): number {
|
||||
return expected === 0 ? 1 : matched / expected;
|
||||
}
|
||||
|
||||
function f1(p: number, r: number): number {
|
||||
if (p + r === 0) return 0;
|
||||
return (2 * p * r) / (p + r);
|
||||
}
|
||||
|
||||
export function aggregate(variants: VariantCase[], results: VariantResult[]): Cat6Report {
|
||||
const byKind = new Map<InjectionKind, { variants: VariantCase[]; results: VariantResult[] }>();
|
||||
for (let i = 0; i < variants.length; i++) {
|
||||
const v = variants[i];
|
||||
const r = results[i];
|
||||
if (!byKind.has(v.kind)) byKind.set(v.kind, { variants: [], results: [] });
|
||||
byKind.get(v.kind)!.variants.push(v);
|
||||
byKind.get(v.kind)!.results.push(r);
|
||||
}
|
||||
|
||||
const per_kind: PerKindReport[] = [];
|
||||
|
||||
let totalMust = 0;
|
||||
let totalMatched = 0;
|
||||
let totalExtracted = 0;
|
||||
let totalFp = 0;
|
||||
let totalFenceCases = 0;
|
||||
let totalFenceLeaks = 0;
|
||||
let totalCollisionCases = 0;
|
||||
let totalCollisionFps = 0;
|
||||
let variantsWithLinks = 0;
|
||||
|
||||
for (const [kind, bucket] of byKind) {
|
||||
let kindMust = 0;
|
||||
let kindMatched = 0;
|
||||
let kindExtracted = 0;
|
||||
let kindFp = 0;
|
||||
|
||||
for (let i = 0; i < bucket.variants.length; i++) {
|
||||
const v = bucket.variants[i];
|
||||
const r = bucket.results[i];
|
||||
kindMust += v.goldDelta.must_extract.length;
|
||||
kindMatched += r.matched;
|
||||
kindExtracted += r.extracted.length;
|
||||
kindFp += r.false_positives.length;
|
||||
if (r.extracted.length > 0) variantsWithLinks++;
|
||||
|
||||
if (kind === 'code_fence_leak') {
|
||||
totalFenceCases += v.goldDelta.must_not_extract.length;
|
||||
totalFenceLeaks += r.false_positives.length;
|
||||
}
|
||||
if (kind === 'substring_collision') {
|
||||
totalCollisionCases += v.goldDelta.must_not_extract.length;
|
||||
totalCollisionFps += r.false_positives.length;
|
||||
}
|
||||
}
|
||||
|
||||
const p = precision(kindMatched, kindExtracted);
|
||||
const r = recall(kindMatched, kindMust);
|
||||
per_kind.push({
|
||||
kind,
|
||||
variants: bucket.variants.length,
|
||||
total_must_extract: kindMust,
|
||||
total_matched: kindMatched,
|
||||
total_false_positives: kindFp,
|
||||
precision: p,
|
||||
recall: r,
|
||||
false_positive_rate: bucket.variants.length === 0 ? 0 : kindFp / bucket.variants.length,
|
||||
});
|
||||
|
||||
totalMust += kindMust;
|
||||
totalMatched += kindMatched;
|
||||
totalExtracted += kindExtracted;
|
||||
totalFp += kindFp;
|
||||
}
|
||||
|
||||
const overallP = precision(totalMatched, totalExtracted);
|
||||
const overallR = recall(totalMatched, totalMust);
|
||||
|
||||
return {
|
||||
schema_version: 1,
|
||||
ran_at: new Date().toISOString(),
|
||||
variants: variants.length,
|
||||
corpus_id: 'world-v1',
|
||||
per_kind,
|
||||
overall: {
|
||||
link_precision: overallP,
|
||||
link_recall: overallR,
|
||||
link_f1: f1(overallP, overallR),
|
||||
code_fence_leak_rate: totalFenceCases === 0 ? 0 : totalFenceLeaks / totalFenceCases,
|
||||
substring_fp_rate: totalCollisionCases === 0 ? 0 : totalCollisionFps / totalCollisionCases,
|
||||
pages_with_links_coverage: variants.length === 0 ? 0 : variantsWithLinks / variants.length,
|
||||
mean_links_per_page: variants.length === 0 ? 0 : totalExtracted / variants.length,
|
||||
},
|
||||
verdict: 'baseline_only',
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Runner entry ─────────────────────────────────────────────────────
|
||||
|
||||
export interface RunCat6Options {
|
||||
corpusDir?: string;
|
||||
perKind?: number;
|
||||
baseSeed?: number;
|
||||
}
|
||||
|
||||
export async function runCat6(opts: RunCat6Options = {}): Promise<Cat6Report> {
|
||||
const pages = loadWorldV1(opts.corpusDir);
|
||||
const variants = generateVariants(pages, {
|
||||
perKind: opts.perKind,
|
||||
baseSeed: opts.baseSeed,
|
||||
});
|
||||
const resolver = makeCorpusResolver(pages);
|
||||
|
||||
const results: VariantResult[] = [];
|
||||
for (const v of variants) {
|
||||
results.push(await scoreVariant(v, resolver));
|
||||
}
|
||||
return aggregate(variants, results);
|
||||
}
|
||||
|
||||
// ─── CLI smoke ────────────────────────────────────────────────────────
|
||||
|
||||
if (import.meta.main) {
|
||||
runCat6()
|
||||
.then(report => {
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
})
|
||||
.catch(err => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
/**
|
||||
* adversarial-injections.ts tests — Day 6 of BrainBench v1 Complete.
|
||||
*
|
||||
* Covers every injection kind:
|
||||
* - Each produces deterministic output under same seed
|
||||
* - goldDelta.must_not_extract lists the adversarial slug for negative injections
|
||||
* - goldDelta.must_extract lists real targets for positive assertions
|
||||
* - applyInjection dispatcher returns the same output as the direct function
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import {
|
||||
applyInjection,
|
||||
injectCodeFenceLeak,
|
||||
injectInlineCodeSlug,
|
||||
injectSubstringCollision,
|
||||
injectAmbiguousRole,
|
||||
injectProseOnlyMention,
|
||||
injectMultiEntitySentence,
|
||||
ALL_INJECTION_KINDS,
|
||||
type EntityRef,
|
||||
} from '../../eval/runner/adversarial-injections.ts';
|
||||
|
||||
const REFS: EntityRef[] = [
|
||||
{ slug: 'people/amara-okafor', name: 'Amara Okafor' },
|
||||
{ slug: 'people/jordan-park', name: 'Jordan Park' },
|
||||
{ slug: 'people/mina-kapoor', name: 'Mina Kapoor' },
|
||||
{ slug: 'people/sarah-chen', name: 'Sarah Chen' },
|
||||
{ slug: 'people/priya-patel', name: 'Priya Patel' },
|
||||
{ slug: 'companies/halfway-capital', name: 'Halfway Capital' },
|
||||
{ slug: 'companies/novamind', name: 'NovaMind' },
|
||||
];
|
||||
|
||||
const BASE_CONTENT = 'This is a fictional biographical page. She is well-known in the climate deals space.';
|
||||
|
||||
// ─── Per-kind assertions ─────────────────────────────────────────────
|
||||
|
||||
describe('injectCodeFenceLeak', () => {
|
||||
test('wraps a fake slug inside triple-backtick fence', () => {
|
||||
const res = injectCodeFenceLeak({ content: BASE_CONTENT, seed: 1, refs: REFS });
|
||||
expect(res.content).toContain('```');
|
||||
const fakeSlug = res.goldDelta.must_not_extract[0].slug;
|
||||
expect(fakeSlug).toMatch(/^people\/fake-\d+$/);
|
||||
expect(res.content).toContain(fakeSlug);
|
||||
});
|
||||
|
||||
test('leaves a real mention outside the fence', () => {
|
||||
const res = injectCodeFenceLeak({ content: BASE_CONTENT, seed: 1, refs: REFS });
|
||||
expect(res.goldDelta.must_extract.length).toBeGreaterThan(0);
|
||||
const realSlug = res.goldDelta.must_extract[0].slug;
|
||||
// Real slug should appear OUTSIDE the fence
|
||||
const fenceEnd = res.content.lastIndexOf('```');
|
||||
const afterFence = res.content.slice(fenceEnd + 3);
|
||||
expect(afterFence).toContain(realSlug);
|
||||
});
|
||||
|
||||
test('deterministic under same seed', () => {
|
||||
const a = injectCodeFenceLeak({ content: BASE_CONTENT, seed: 42, refs: REFS });
|
||||
const b = injectCodeFenceLeak({ content: BASE_CONTENT, seed: 42, refs: REFS });
|
||||
expect(a.content).toEqual(b.content);
|
||||
});
|
||||
});
|
||||
|
||||
describe('injectInlineCodeSlug', () => {
|
||||
test('wraps slug in single-backtick inline code', () => {
|
||||
const res = injectInlineCodeSlug({ content: BASE_CONTENT, seed: 1, refs: REFS });
|
||||
const fakeSlug = res.goldDelta.must_not_extract[0].slug;
|
||||
expect(res.content).toContain(`\`${fakeSlug}\``);
|
||||
});
|
||||
});
|
||||
|
||||
describe('injectSubstringCollision', () => {
|
||||
test('injects a "<Name>AI" substring near a real mention', () => {
|
||||
const forced: EntityRef[] = [{ slug: 'people/sam', name: 'Sam' }];
|
||||
const res = injectSubstringCollision({ content: BASE_CONTENT, seed: 1, refs: REFS, forcedRefs: forced });
|
||||
expect(res.content).toContain('SamAI');
|
||||
expect(res.content).toContain('[Sam](people/sam)');
|
||||
expect(res.goldDelta.must_not_extract[0].slug).toBe('people/samai');
|
||||
});
|
||||
});
|
||||
|
||||
describe('injectAmbiguousRole', () => {
|
||||
test('replaces "works at" with "works with" when present', () => {
|
||||
const input = 'Alice works at [Acme](companies/acme).';
|
||||
const res = injectAmbiguousRole({ content: input, seed: 1, refs: REFS });
|
||||
expect(res.content).toContain('works with');
|
||||
expect(res.content).not.toContain('works at');
|
||||
});
|
||||
|
||||
test('appends a works-with sentence when source has no "works at"', () => {
|
||||
const res = injectAmbiguousRole({ content: 'plain prose.', seed: 1, refs: REFS });
|
||||
expect(res.content).toContain('works with');
|
||||
});
|
||||
|
||||
test('goldDelta says type must be mentions, not works_at', () => {
|
||||
const res = injectAmbiguousRole({ content: 'plain prose.', seed: 1, refs: REFS });
|
||||
expect(res.goldDelta.must_extract[0].type).toBe('mentions');
|
||||
});
|
||||
});
|
||||
|
||||
describe('injectProseOnlyMention', () => {
|
||||
test('strips [name](slug) syntax leaving bare name', () => {
|
||||
const forced: EntityRef[] = [{ slug: 'people/jordan-park', name: 'Jordan Park' }];
|
||||
const input = 'I met [Jordan Park](people/jordan-park) yesterday.';
|
||||
const res = injectProseOnlyMention({ content: input, seed: 1, refs: REFS, forcedRefs: forced });
|
||||
expect(res.content).not.toContain('[Jordan Park]');
|
||||
expect(res.content).not.toContain('(people/jordan-park)');
|
||||
expect(res.content).toContain('Jordan Park');
|
||||
});
|
||||
|
||||
test('goldDelta requires extraction as mentions', () => {
|
||||
const forced: EntityRef[] = [{ slug: 'people/jordan-park', name: 'Jordan Park' }];
|
||||
const input = 'I met [Jordan Park](people/jordan-park) yesterday.';
|
||||
const res = injectProseOnlyMention({ content: input, seed: 1, refs: REFS, forcedRefs: forced });
|
||||
expect(res.goldDelta.must_extract[0]).toEqual({
|
||||
slug: 'people/jordan-park',
|
||||
type: 'mentions',
|
||||
reason: expect.any(String),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('injectMultiEntitySentence', () => {
|
||||
test('packs 4+ entities into one clause', () => {
|
||||
const res = injectMultiEntitySentence({ content: BASE_CONTENT, seed: 1, refs: REFS });
|
||||
expect(res.goldDelta.must_extract.length).toBeGreaterThanOrEqual(4);
|
||||
// Every must_extract slug should appear in content as markdown link
|
||||
for (const m of res.goldDelta.must_extract) {
|
||||
expect(res.content).toContain(`](${m.slug})`);
|
||||
}
|
||||
});
|
||||
|
||||
test('skips gracefully when fewer than 4 refs are available', () => {
|
||||
const res = injectMultiEntitySentence({
|
||||
content: BASE_CONTENT,
|
||||
seed: 1,
|
||||
refs: REFS.slice(0, 2),
|
||||
});
|
||||
expect(res.goldDelta.must_extract.length).toBe(0);
|
||||
expect(res.goldDelta.note).toContain('skipped');
|
||||
});
|
||||
|
||||
test('picks are unique (no duplicate slugs in one clause)', () => {
|
||||
const res = injectMultiEntitySentence({ content: BASE_CONTENT, seed: 1, refs: REFS });
|
||||
const slugs = res.goldDelta.must_extract.map(m => m.slug);
|
||||
expect(new Set(slugs).size).toBe(slugs.length);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Dispatcher + kind coverage ──────────────────────────────────────
|
||||
|
||||
describe('applyInjection dispatcher', () => {
|
||||
test('ALL_INJECTION_KINDS lists exactly 6 kinds', () => {
|
||||
expect(ALL_INJECTION_KINDS.length).toBe(6);
|
||||
});
|
||||
|
||||
test('dispatcher produces same output as the direct function (code_fence_leak)', () => {
|
||||
const direct = injectCodeFenceLeak({ content: BASE_CONTENT, seed: 5, refs: REFS });
|
||||
const viaDispatch = applyInjection('code_fence_leak', { content: BASE_CONTENT, seed: 5, refs: REFS });
|
||||
expect(viaDispatch.content).toBe(direct.content);
|
||||
});
|
||||
|
||||
test('every kind produces a non-empty note in goldDelta', () => {
|
||||
for (const kind of ALL_INJECTION_KINDS) {
|
||||
const res = applyInjection(kind, { content: BASE_CONTENT, seed: 7, refs: REFS });
|
||||
expect(res.goldDelta.note.length).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
test('every kind produces valid slug format in gold entries', () => {
|
||||
const SLUG_RE = /^[a-z][a-z0-9-]*\/[a-z0-9][a-z0-9-]*$/;
|
||||
for (const kind of ALL_INJECTION_KINDS) {
|
||||
const res = applyInjection(kind, { content: BASE_CONTENT, seed: 9, refs: REFS });
|
||||
for (const m of res.goldDelta.must_extract) expect(m.slug).toMatch(SLUG_RE);
|
||||
for (const m of res.goldDelta.must_not_extract) {
|
||||
// Fake slugs are built to resemble real slug shape
|
||||
expect(m.slug).toMatch(SLUG_RE);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,316 @@
|
||||
/**
|
||||
* cat11-multimodal.ts tests — Day 6 of BrainBench v1 Complete.
|
||||
*
|
||||
* Covers:
|
||||
* - charSimilarity math: identical / empty / partial
|
||||
* - wordRecall: multiset semantics, empty canonical
|
||||
* - wer (word error rate): perfect / half-off / empty ref
|
||||
* - htmlToText: strips script/style/tags, decodes common entities
|
||||
* - runHtmlModality with a real fixture (tmpdir, hand-rolled HTML)
|
||||
* - runAudioModality skips gracefully without API keys + manifest
|
||||
* - runPdfModality skips gracefully without manifest
|
||||
* - runAudioModality uses injected transcriber stub
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import { mkdirSync, writeFileSync, rmSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
import {
|
||||
charSimilarity,
|
||||
wordRecall,
|
||||
wer,
|
||||
htmlToText,
|
||||
runHtmlModality,
|
||||
runAudioModality,
|
||||
runPdfModality,
|
||||
runCat11,
|
||||
} from '../../eval/runner/cat11-multimodal.ts';
|
||||
|
||||
function tmpFixturesRoot(): string {
|
||||
const dir = join(tmpdir(), `cat11-test-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`);
|
||||
mkdirSync(dir, { recursive: true });
|
||||
return dir;
|
||||
}
|
||||
|
||||
// ─── Pure math helpers ────────────────────────────────────────────────
|
||||
|
||||
describe('charSimilarity', () => {
|
||||
test('identical strings → 1', () => {
|
||||
expect(charSimilarity('hello world', 'hello world')).toBe(1);
|
||||
});
|
||||
|
||||
test('empty vs non-empty → 0', () => {
|
||||
expect(charSimilarity('', 'hello')).toBe(0);
|
||||
expect(charSimilarity('hello', '')).toBe(0);
|
||||
});
|
||||
|
||||
test('both empty → 1', () => {
|
||||
expect(charSimilarity('', '')).toBe(1);
|
||||
});
|
||||
|
||||
test('one-char substitution', () => {
|
||||
// 'cat' vs 'bat' = 1 edit over max 3 chars = 1 - 1/3 = 0.667
|
||||
expect(charSimilarity('cat', 'bat')).toBeCloseTo(2 / 3, 6);
|
||||
});
|
||||
});
|
||||
|
||||
describe('wordRecall', () => {
|
||||
test('all canonical words present → 1', () => {
|
||||
expect(wordRecall('the quick brown fox', 'the lazy fox jumped over the quick brown dog')).toBe(1);
|
||||
});
|
||||
|
||||
test('none present → 0', () => {
|
||||
expect(wordRecall('one two three', 'alpha beta gamma')).toBe(0);
|
||||
});
|
||||
|
||||
test('empty canonical → 1 (trivially satisfied)', () => {
|
||||
expect(wordRecall('', 'anything')).toBe(1);
|
||||
});
|
||||
|
||||
test('multiset semantics: three "the" vs one "the" → 1/3', () => {
|
||||
expect(wordRecall('the the the', 'the cat')).toBeCloseTo(1 / 3, 6);
|
||||
});
|
||||
|
||||
test('case-insensitive + strips punctuation', () => {
|
||||
expect(wordRecall('Hello, world!', 'hello world')).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('wer', () => {
|
||||
test('perfect transcription → 0', () => {
|
||||
expect(wer('hello world', 'hello world')).toBe(0);
|
||||
});
|
||||
|
||||
test('one word wrong → 0.5 on 2-word reference', () => {
|
||||
expect(wer('hello world', 'hello there')).toBe(0.5);
|
||||
});
|
||||
|
||||
test('empty reference vs empty hypothesis → 0', () => {
|
||||
expect(wer('', '')).toBe(0);
|
||||
});
|
||||
|
||||
test('empty reference + non-empty hypothesis → 1', () => {
|
||||
expect(wer('', 'anything')).toBe(1);
|
||||
});
|
||||
|
||||
test('completely wrong → 1', () => {
|
||||
expect(wer('one two three', 'alpha beta gamma')).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('htmlToText', () => {
|
||||
test('strips tags', () => {
|
||||
expect(htmlToText('<p>hello <b>world</b></p>')).toBe('hello world');
|
||||
});
|
||||
|
||||
test('drops script + style contents entirely', () => {
|
||||
const html = '<html><head><style>body{}</style><script>alert(1)</script></head><body>visible</body></html>';
|
||||
const out = htmlToText(html);
|
||||
expect(out).toContain('visible');
|
||||
expect(out).not.toContain('alert');
|
||||
expect(out).not.toContain('body{}');
|
||||
});
|
||||
|
||||
test('decodes common entities', () => {
|
||||
expect(htmlToText('<tag> & "quote"')).toContain('<tag> & "quote"');
|
||||
});
|
||||
|
||||
test('collapses whitespace', () => {
|
||||
expect(htmlToText('<p>a\n\nb \t c</p>')).toBe('a b c');
|
||||
});
|
||||
});
|
||||
|
||||
// ─── runHtmlModality (real filesystem + real manifest) ────────────────
|
||||
|
||||
describe('runHtmlModality', () => {
|
||||
let root: string;
|
||||
beforeEach(() => {
|
||||
root = tmpFixturesRoot();
|
||||
});
|
||||
afterEach(() => {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test('skips gracefully when manifest missing', async () => {
|
||||
const result = await runHtmlModality(root);
|
||||
expect(result.skipped).toBe(true);
|
||||
expect(result.skip_reason).toContain('manifest');
|
||||
});
|
||||
|
||||
test('scores real HTML fixture against canonical', async () => {
|
||||
writeFileSync(join(root, 'page.html'), '<html><body><p>hello world</p></body></html>');
|
||||
writeFileSync(join(root, 'page.txt'), 'hello world');
|
||||
writeFileSync(
|
||||
join(root, 'fixtures.json'),
|
||||
JSON.stringify({
|
||||
version: 1,
|
||||
license: 'CC-BY-SA',
|
||||
items: [
|
||||
{
|
||||
name: 'page',
|
||||
path: 'page.html',
|
||||
canonical_path: 'page.txt',
|
||||
sha256: 'unused-in-test',
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
const result = await runHtmlModality(root);
|
||||
expect(result.skipped).toBe(false);
|
||||
expect(result.items).toBe(1);
|
||||
expect(result.items_attempted).toBe(1);
|
||||
expect(result.mean_metric).toBe(1); // canonical "hello world" recovered exactly
|
||||
expect(result.per_item[0].name).toBe('page');
|
||||
});
|
||||
|
||||
test('missing fixture file produces an error entry (not a crash)', async () => {
|
||||
writeFileSync(
|
||||
join(root, 'fixtures.json'),
|
||||
JSON.stringify({
|
||||
version: 1,
|
||||
license: 'CC-BY-SA',
|
||||
items: [{ name: 'ghost', path: 'ghost.html', canonical_path: 'ghost.txt', sha256: 'x' }],
|
||||
}),
|
||||
);
|
||||
const result = await runHtmlModality(root);
|
||||
expect(result.skipped).toBe(false);
|
||||
expect(result.per_item[0].error).toContain('missing');
|
||||
expect(result.items_attempted).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── runAudioModality ─────────────────────────────────────────────────
|
||||
|
||||
describe('runAudioModality', () => {
|
||||
let root: string;
|
||||
const originalGroq = process.env.GROQ_API_KEY;
|
||||
const originalOpenAI = process.env.OPENAI_API_KEY;
|
||||
|
||||
beforeEach(() => {
|
||||
root = tmpFixturesRoot();
|
||||
delete process.env.GROQ_API_KEY;
|
||||
delete process.env.OPENAI_API_KEY;
|
||||
});
|
||||
afterEach(() => {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
if (originalGroq !== undefined) process.env.GROQ_API_KEY = originalGroq;
|
||||
if (originalOpenAI !== undefined) process.env.OPENAI_API_KEY = originalOpenAI;
|
||||
});
|
||||
|
||||
test('skips gracefully when manifest missing', async () => {
|
||||
const result = await runAudioModality(root);
|
||||
expect(result.skipped).toBe(true);
|
||||
expect(result.skip_reason).toContain('manifest');
|
||||
});
|
||||
|
||||
test('skips gracefully when manifest present but no API key + no stub', async () => {
|
||||
writeFileSync(
|
||||
join(root, 'fixtures.json'),
|
||||
JSON.stringify({
|
||||
version: 1,
|
||||
license: 'CC0',
|
||||
items: [{ name: 'clip', path: 'clip.mp3', canonical_path: 'clip.txt', sha256: 'x' }],
|
||||
}),
|
||||
);
|
||||
const result = await runAudioModality(root);
|
||||
expect(result.skipped).toBe(true);
|
||||
expect(result.skip_reason).toContain('GROQ_API_KEY');
|
||||
});
|
||||
|
||||
test('uses injected transcriber stub and computes WER', async () => {
|
||||
writeFileSync(join(root, 'clip.mp3'), 'fake audio bytes');
|
||||
writeFileSync(join(root, 'clip.txt'), 'hello world this is a test');
|
||||
writeFileSync(
|
||||
join(root, 'fixtures.json'),
|
||||
JSON.stringify({
|
||||
version: 1,
|
||||
license: 'CC0',
|
||||
items: [{ name: 'clip', path: 'clip.mp3', canonical_path: 'clip.txt', sha256: 'x' }],
|
||||
}),
|
||||
);
|
||||
const stubTranscribe = async () => ({ text: 'hello world this is a test', provider: 'stub' });
|
||||
const result = await runAudioModality(root, { transcribe: stubTranscribe });
|
||||
expect(result.skipped).toBe(false);
|
||||
expect(result.mean_metric).toBe(0); // perfect transcription → WER 0
|
||||
});
|
||||
|
||||
test('stub that returns wrong transcription produces non-zero WER', async () => {
|
||||
writeFileSync(join(root, 'clip.mp3'), 'bytes');
|
||||
writeFileSync(join(root, 'clip.txt'), 'one two three four');
|
||||
writeFileSync(
|
||||
join(root, 'fixtures.json'),
|
||||
JSON.stringify({
|
||||
version: 1,
|
||||
license: 'CC0',
|
||||
items: [{ name: 'clip', path: 'clip.mp3', canonical_path: 'clip.txt', sha256: 'x' }],
|
||||
}),
|
||||
);
|
||||
const stub = async () => ({ text: 'one two three five', provider: 'stub' });
|
||||
const result = await runAudioModality(root, { transcribe: stub });
|
||||
expect(result.skipped).toBe(false);
|
||||
expect(result.mean_metric).toBe(0.25); // 1 word wrong / 4 words = 0.25
|
||||
});
|
||||
});
|
||||
|
||||
// ─── runPdfModality ───────────────────────────────────────────────────
|
||||
|
||||
describe('runPdfModality', () => {
|
||||
let root: string;
|
||||
beforeEach(() => { root = tmpFixturesRoot(); });
|
||||
afterEach(() => { rmSync(root, { recursive: true, force: true }); });
|
||||
|
||||
test('skips gracefully when manifest missing', async () => {
|
||||
const result = await runPdfModality(root);
|
||||
expect(result.skipped).toBe(true);
|
||||
expect(result.skip_reason).toContain('manifest');
|
||||
});
|
||||
|
||||
test('missing PDF file produces error entry (not crash)', async () => {
|
||||
writeFileSync(
|
||||
join(root, 'fixtures.json'),
|
||||
JSON.stringify({
|
||||
version: 1,
|
||||
license: 'CC-BY',
|
||||
items: [{ name: 'paper', path: 'ghost.pdf', canonical_path: 'ghost.txt', sha256: 'x' }],
|
||||
}),
|
||||
);
|
||||
const result = await runPdfModality(root);
|
||||
expect(result.skipped).toBe(false);
|
||||
expect(result.per_item[0].error).toContain('missing');
|
||||
});
|
||||
});
|
||||
|
||||
// ─── runCat11 full dispatch ───────────────────────────────────────────
|
||||
|
||||
describe('runCat11', () => {
|
||||
let root: string;
|
||||
beforeEach(() => { root = tmpFixturesRoot(); });
|
||||
afterEach(() => { rmSync(root, { recursive: true, force: true }); });
|
||||
|
||||
test('returns skipped verdict when no modality has fixtures', async () => {
|
||||
const report = await runCat11({ fixturesRoot: root });
|
||||
expect(report.verdict).toBe('skipped');
|
||||
expect(report.results.pdf.skipped).toBe(true);
|
||||
expect(report.results.html.skipped).toBe(true);
|
||||
expect(report.results.audio.skipped).toBe(true);
|
||||
});
|
||||
|
||||
test('returns baseline_only when at least one modality ran', async () => {
|
||||
// Set up an HTML fixture so at least one modality runs
|
||||
mkdirSync(join(root, 'html'), { recursive: true });
|
||||
writeFileSync(join(root, 'html/p.html'), '<body>hello</body>');
|
||||
writeFileSync(join(root, 'html/p.txt'), 'hello');
|
||||
writeFileSync(
|
||||
join(root, 'html/fixtures.json'),
|
||||
JSON.stringify({
|
||||
version: 1,
|
||||
license: 'CC-BY-SA',
|
||||
items: [{ name: 'p', path: 'p.html', canonical_path: 'p.txt', sha256: 'x' }],
|
||||
}),
|
||||
);
|
||||
const report = await runCat11({ fixturesRoot: root });
|
||||
expect(report.verdict).toBe('baseline_only');
|
||||
expect(report.results.html.skipped).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,202 @@
|
||||
/**
|
||||
* cat6-prose-scale.ts tests — Day 6 of BrainBench v1 Complete.
|
||||
*
|
||||
* Uses a tiny synthetic corpus (5 pages) so tests run in <200ms, not the
|
||||
* full 240-page world-v1 which is expensive to traverse per-test.
|
||||
*
|
||||
* Covers:
|
||||
* - generateVariants produces deterministic variantIds under fixed seed
|
||||
* - Variants cover all 6 injection kinds
|
||||
* - scoreVariant computes matched/missed/false_positives correctly
|
||||
* - aggregate produces all per-kind metrics + overall numbers
|
||||
* - Verdict is always 'baseline_only' in v1 (no gating threshold)
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import {
|
||||
generateVariants,
|
||||
aggregate,
|
||||
scoreVariant,
|
||||
makeCorpusResolver,
|
||||
type BasePage,
|
||||
} from '../../eval/runner/cat6-prose-scale.ts';
|
||||
import { ALL_INJECTION_KINDS } from '../../eval/runner/adversarial-injections.ts';
|
||||
|
||||
const TINY_CORPUS: BasePage[] = [
|
||||
{
|
||||
slug: 'people/amara',
|
||||
type: 'person',
|
||||
title: 'Amara Okafor',
|
||||
content: 'Amara is a partner at [Halfway](companies/halfway).',
|
||||
},
|
||||
{
|
||||
slug: 'people/jordan',
|
||||
type: 'person',
|
||||
title: 'Jordan Park',
|
||||
content: 'Jordan founded [NovaMind](companies/novamind) in 2023.',
|
||||
},
|
||||
{
|
||||
slug: 'people/mina',
|
||||
type: 'person',
|
||||
title: 'Mina Kapoor',
|
||||
content: 'Mina runs [Threshold](companies/threshold).',
|
||||
},
|
||||
{
|
||||
slug: 'people/sarah',
|
||||
type: 'person',
|
||||
title: 'Sarah Chen',
|
||||
content: 'Sarah advises several seed-stage founders.',
|
||||
},
|
||||
{
|
||||
slug: 'companies/halfway',
|
||||
type: 'company',
|
||||
title: 'Halfway Capital',
|
||||
content: 'VC firm focused on climate + AI infrastructure.',
|
||||
},
|
||||
{
|
||||
slug: 'companies/novamind',
|
||||
type: 'company',
|
||||
title: 'NovaMind',
|
||||
content: 'AI infrastructure startup.',
|
||||
},
|
||||
{
|
||||
slug: 'companies/threshold',
|
||||
type: 'company',
|
||||
title: 'Threshold',
|
||||
content: 'Venture firm.',
|
||||
},
|
||||
];
|
||||
|
||||
// ─── Variant generation ───────────────────────────────────────────────
|
||||
|
||||
describe('generateVariants', () => {
|
||||
test('produces exactly perKind × 6 variants', () => {
|
||||
const variants = generateVariants(TINY_CORPUS, { perKind: 3 });
|
||||
expect(variants.length).toBe(3 * ALL_INJECTION_KINDS.length);
|
||||
});
|
||||
|
||||
test('variants are distributed across all kinds', () => {
|
||||
const variants = generateVariants(TINY_CORPUS, { perKind: 2 });
|
||||
const kinds = new Set(variants.map(v => v.kind));
|
||||
expect(kinds.size).toBe(ALL_INJECTION_KINDS.length);
|
||||
});
|
||||
|
||||
test('deterministic under fixed seed', () => {
|
||||
const a = generateVariants(TINY_CORPUS, { perKind: 2, baseSeed: 42 });
|
||||
const b = generateVariants(TINY_CORPUS, { perKind: 2, baseSeed: 42 });
|
||||
expect(a.length).toBe(b.length);
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
expect(a[i].variantId).toBe(b[i].variantId);
|
||||
expect(a[i].content).toBe(b[i].content);
|
||||
}
|
||||
});
|
||||
|
||||
test('different seeds produce different content', () => {
|
||||
const a = generateVariants(TINY_CORPUS, { perKind: 2, baseSeed: 42 });
|
||||
const b = generateVariants(TINY_CORPUS, { perKind: 2, baseSeed: 99 });
|
||||
// code_fence_leak's fake slug depends on seed; content will differ.
|
||||
const aCodeFences = a.filter(v => v.kind === 'code_fence_leak');
|
||||
const bCodeFences = b.filter(v => v.kind === 'code_fence_leak');
|
||||
expect(aCodeFences[0].content).not.toBe(bCodeFences[0].content);
|
||||
});
|
||||
|
||||
test('variantId follows "<slug>-v<idx>-<kind>" pattern', () => {
|
||||
const variants = generateVariants(TINY_CORPUS, { perKind: 1 });
|
||||
for (const v of variants) {
|
||||
expect(v.variantId).toMatch(/^.+-v\d+-[a-z_]+$/);
|
||||
expect(v.variantId.endsWith(`-${v.kind}`)).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ─── scoreVariant ─────────────────────────────────────────────────────
|
||||
|
||||
describe('scoreVariant', () => {
|
||||
test('returns VariantResult with extracted, missed, false_positives, matched', async () => {
|
||||
const resolver = makeCorpusResolver(TINY_CORPUS);
|
||||
const variants = generateVariants(TINY_CORPUS, { perKind: 1 });
|
||||
const result = await scoreVariant(variants[0], resolver);
|
||||
expect(result.variantId).toBe(variants[0].variantId);
|
||||
expect(Array.isArray(result.extracted)).toBe(true);
|
||||
expect(Array.isArray(result.false_positives)).toBe(true);
|
||||
expect(Array.isArray(result.missed)).toBe(true);
|
||||
expect(typeof result.matched).toBe('number');
|
||||
});
|
||||
|
||||
test('counts a must_extract slug as matched when extractor produces it', async () => {
|
||||
// Multi-entity sentences pack ≥4 real refs and the resolver accepts all known slugs.
|
||||
// The extractor should produce those links.
|
||||
const resolver = makeCorpusResolver(TINY_CORPUS);
|
||||
const variants = generateVariants(TINY_CORPUS, { perKind: 5 });
|
||||
const multi = variants.find(v => v.kind === 'multi_entity_sentence' && v.goldDelta.must_extract.length >= 4);
|
||||
if (!multi) {
|
||||
// Skip gracefully if TINY_CORPUS is too small
|
||||
return;
|
||||
}
|
||||
const result = await scoreVariant(multi, resolver);
|
||||
expect(result.matched).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── aggregate ────────────────────────────────────────────────────────
|
||||
|
||||
describe('aggregate', () => {
|
||||
test('emits per_kind for every kind present', async () => {
|
||||
const resolver = makeCorpusResolver(TINY_CORPUS);
|
||||
const variants = generateVariants(TINY_CORPUS, { perKind: 2 });
|
||||
const results = await Promise.all(variants.map(v => scoreVariant(v, resolver)));
|
||||
const report = aggregate(variants, results);
|
||||
expect(report.per_kind.length).toBe(ALL_INJECTION_KINDS.length);
|
||||
expect(report.variants).toBe(variants.length);
|
||||
});
|
||||
|
||||
test('overall metrics are computed (precision/recall/f1 in 0-1 range)', async () => {
|
||||
const resolver = makeCorpusResolver(TINY_CORPUS);
|
||||
const variants = generateVariants(TINY_CORPUS, { perKind: 2 });
|
||||
const results = await Promise.all(variants.map(v => scoreVariant(v, resolver)));
|
||||
const report = aggregate(variants, results);
|
||||
expect(report.overall.link_precision).toBeGreaterThanOrEqual(0);
|
||||
expect(report.overall.link_precision).toBeLessThanOrEqual(1);
|
||||
expect(report.overall.link_recall).toBeGreaterThanOrEqual(0);
|
||||
expect(report.overall.link_recall).toBeLessThanOrEqual(1);
|
||||
expect(report.overall.link_f1).toBeGreaterThanOrEqual(0);
|
||||
expect(report.overall.link_f1).toBeLessThanOrEqual(1);
|
||||
});
|
||||
|
||||
test('verdict is always baseline_only in v1', async () => {
|
||||
const resolver = makeCorpusResolver(TINY_CORPUS);
|
||||
const variants = generateVariants(TINY_CORPUS, { perKind: 1 });
|
||||
const results = await Promise.all(variants.map(v => scoreVariant(v, resolver)));
|
||||
const report = aggregate(variants, results);
|
||||
expect(report.verdict).toBe('baseline_only');
|
||||
});
|
||||
|
||||
test('mean_links_per_page matches totalExtracted / variants', async () => {
|
||||
const resolver = makeCorpusResolver(TINY_CORPUS);
|
||||
const variants = generateVariants(TINY_CORPUS, { perKind: 1 });
|
||||
const results = await Promise.all(variants.map(v => scoreVariant(v, resolver)));
|
||||
const report = aggregate(variants, results);
|
||||
const totalExtracted = results.reduce((sum, r) => sum + r.extracted.length, 0);
|
||||
expect(report.overall.mean_links_per_page).toBeCloseTo(totalExtracted / variants.length, 6);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Corpus resolver ──────────────────────────────────────────────────
|
||||
|
||||
describe('makeCorpusResolver', () => {
|
||||
test('returns slug unchanged when slug is known', async () => {
|
||||
const resolver = makeCorpusResolver(TINY_CORPUS);
|
||||
expect(await resolver.resolve('people/amara')).toBe('people/amara');
|
||||
});
|
||||
|
||||
test('returns null when slug is unknown', async () => {
|
||||
const resolver = makeCorpusResolver(TINY_CORPUS);
|
||||
expect(await resolver.resolve('people/ghost')).toBeNull();
|
||||
});
|
||||
|
||||
test('does not resolve bare names in v1', async () => {
|
||||
const resolver = makeCorpusResolver(TINY_CORPUS);
|
||||
// Cat 6 scoring semantics: bare-name resolution is out of scope.
|
||||
expect(await resolver.resolve('Amara Okafor')).toBeNull();
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user