diff --git a/eval/runner/multi-adapter.ts b/eval/runner/multi-adapter.ts index 4aa4782c3..70e505746 100644 --- a/eval/runner/multi-adapter.ts +++ b/eval/runner/multi-adapter.ts @@ -26,7 +26,7 @@ import { RipgrepBm25Adapter } from './adapters/ripgrep-bm25.ts'; import { VectorOnlyAdapter } from './adapters/vector-only.ts'; import { HybridNoGraphAdapter } from './adapters/hybrid-nograph.ts'; import type { Adapter, Page, Query, RankedDoc } from './types.ts'; -import { precisionAtK, recallAtK } from './types.ts'; +import { precisionAtK, recallAtK, sanitizePage, sanitizeQuery } from './types.ts'; const TOP_K = 5; @@ -331,13 +331,19 @@ async function scoreOneRun( pages: Page[], queries: Query[], ): Promise { - const state = await adapter.init(pages, { name: adapter.name }); + // Day 9 sealed qrels enforcement (codex fix #1, #2, #3): + // Build sanitized copies with no `_facts` and no `gold` fields before + // handing them to the adapter. The scorer retains the full Query shape + // (including gold.relevant) to compute precision/recall below. + const publicPages = pages.map(sanitizePage); + const state = await adapter.init(publicPages, { name: adapter.name }); let totalP = 0; let totalR = 0; let totalCorrect = 0; let totalExpected = 0; for (const q of queries) { - const results = await adapter.query(q, state); + const publicQ = sanitizeQuery(q); + const results = await adapter.query(publicQ as unknown as Query, state); const relevant = new Set(q.gold.relevant ?? []); totalP += precisionAtK(results, relevant, TOP_K); totalR += recallAtK(results, relevant, TOP_K); diff --git a/eval/runner/types.ts b/eval/runner/types.ts index 325e22e1c..94017d9fa 100644 --- a/eval/runner/types.ts +++ b/eval/runner/types.ts @@ -39,6 +39,30 @@ export interface Page { frontmatter?: Record; } +/** + * PublicPage — the shape adapters SEE at runtime (Day 9 sealed-qrels). + * + * Multi-adapter.ts calls `sanitizePage()` before passing the array to + * `adapter.init()`. The sanitized copy has NO `_facts`, NO `frontmatter`, + * NO arbitrary keys — only the five public fields below. Adapters that + * try `(page as any)._facts` get `undefined` instead of the gold object. + * + * This is soft enforcement (a misbehaving adapter could still open + * `eval/data/gold/*.json` from disk). Hard enforcement via process + * isolation ships with BrainBench v2's Docker sandbox. + */ +export type PublicPage = Pick; + +export function sanitizePage(p: Page): PublicPage { + return { + slug: p.slug, + type: p.type, + title: p.title, + compiled_truth: p.compiled_truth, + timeline: p.timeline, + }; +} + // ─── Query ─────────────────────────────────────────────────────────── export type Tier = @@ -89,6 +113,37 @@ export interface Query { tags?: string[]; // 'identity-collision', 'contradiction', etc. } +/** + * PublicQuery — the shape adapters SEE at runtime (Day 9 sealed-qrels). + * + * Multi-adapter.ts calls `sanitizeQuery()` before passing each query to + * `adapter.query()`. The sanitized copy strips the `gold` field entirely, + * so an adapter cannot read `q.gold.relevant` to cheat. Scorers keep the + * full Query shape and compare adapter output against gold after the call. + * + * Adapters that need as_of_date for temporal queries still get it — + * PublicQuery keeps every field EXCEPT gold. + */ +export type PublicQuery = Omit; + +export function sanitizeQuery(q: Query): PublicQuery { + // Build a new object to sever the reference chain. Using spread + delete + // would leave the `gold` key on the prototype-shape in some engines; + // explicit enumeration is the safest pattern. + const out: PublicQuery = { + id: q.id, + tier: q.tier, + text: q.text, + expected_output_type: q.expected_output_type, + }; + if (q.as_of_date !== undefined) out.as_of_date = q.as_of_date; + if (q.acceptable_variants !== undefined) out.acceptable_variants = q.acceptable_variants; + if (q.known_failure_modes !== undefined) out.known_failure_modes = q.known_failure_modes; + if (q.author !== undefined) out.author = q.author; + if (q.tags !== undefined) out.tags = q.tags; + return out; +} + // ─── RankedDoc ────────────────────────────────────────────────────── /** diff --git a/test/eval/sealed-qrels.test.ts b/test/eval/sealed-qrels.test.ts new file mode 100644 index 000000000..a9969a628 --- /dev/null +++ b/test/eval/sealed-qrels.test.ts @@ -0,0 +1,257 @@ +/** + * sealed-qrels regression test — Day 9 of BrainBench v1 Complete. + * + * Enforces the sealed-qrels contract added in Day 9: + * - sanitizePage() produces a new object with NO `_facts` field + * - sanitizeQuery() produces a new object with NO `gold` field + * - Accessing `._facts` / `.gold` on sanitized output returns `undefined` + * - Scorer retains the full Query/RichPage shape (gold.relevant still usable) + * + * This is a SOFT enforcement — an adapter that runs `readFileSync( + * 'eval/data/gold/*.json')` could still cheat. Hard enforcement via + * process isolation ships with BrainBench v2's Docker sandbox. + * + * Documented as such so the adversarial reviewer doesn't get a false sense + * of airtight enforcement here. + */ + +import { describe, test, expect } from 'bun:test'; +import { + sanitizePage, + sanitizeQuery, + type Page, + type PublicPage, + type Query, + type PublicQuery, +} from '../../eval/runner/types.ts'; + +// ─── RichPage helper (mirrors multi-adapter.ts internal shape) ──────── + +interface RichPage extends Page { + _facts: { + type: string; + attendees?: string[]; + employees?: string[]; + founders?: string[]; + investors?: string[]; + }; +} + +function makeRichPage(overrides: Partial = {}): RichPage { + return { + slug: 'people/amara', + type: 'person', + title: 'Amara Okafor', + compiled_truth: 'Amara is a Partner.', + timeline: '', + _facts: { type: 'person' }, + ...overrides, + } as RichPage; +} + +function makeQuery(overrides: Partial = {}): Query { + return { + id: 'q-0001', + tier: 'easy', + text: 'Who is Amara?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/amara'] }, + ...overrides, + }; +} + +// ─── sanitizePage ───────────────────────────────────────────────────── + +describe('sanitizePage — strips _facts and frontmatter', () => { + test('output has the 5 public fields', () => { + const rp = makeRichPage(); + const sanitized = sanitizePage(rp); + expect(sanitized.slug).toBe(rp.slug); + expect(sanitized.type).toBe(rp.type); + expect(sanitized.title).toBe(rp.title); + expect(sanitized.compiled_truth).toBe(rp.compiled_truth); + expect(sanitized.timeline).toBe(rp.timeline); + }); + + test('output does NOT have _facts (the gold canonical leak)', () => { + const rp = makeRichPage({ + _facts: { type: 'person', employees: ['people/amara'] }, + }); + const sanitized = sanitizePage(rp); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + expect((sanitized as any)._facts).toBeUndefined(); + expect('_facts' in sanitized).toBe(false); + }); + + test('output does NOT have frontmatter (potential hiding spot)', () => { + const rp = makeRichPage(); + // Caller could have dumped _facts into frontmatter as a workaround + rp.frontmatter = { _facts_leak: 'gold data' }; + const sanitized = sanitizePage(rp); + expect('frontmatter' in sanitized).toBe(false); + }); + + test('output is a NEW object (not a reference to the original)', () => { + const rp = makeRichPage(); + const sanitized = sanitizePage(rp); + expect(sanitized).not.toBe(rp as unknown as PublicPage); + }); + + test('output has exactly the 5 expected keys (no hidden properties)', () => { + const rp = makeRichPage(); + rp._facts = { type: 'person', employees: ['x'] }; + rp.frontmatter = { anything: 'goes' }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (rp as any).leak = 'secret'; + const sanitized = sanitizePage(rp); + const keys = Object.keys(sanitized).sort(); + expect(keys).toEqual(['compiled_truth', 'slug', 'timeline', 'title', 'type']); + }); + + test('sanitized page when cast to any cannot reach original _facts', () => { + const rp = makeRichPage({ + _facts: { type: 'company', investors: ['people/alice'] }, + }); + const sanitized = sanitizePage(rp); + // A cheating adapter does: const x = (page as any)._facts; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const facts = (sanitized as any)._facts; + expect(facts).toBeUndefined(); + // And cannot reach the original by prototype chain either + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const proto = Object.getPrototypeOf(sanitized); + expect(proto).toBe(Object.prototype); + }); +}); + +// ─── sanitizeQuery ──────────────────────────────────────────────────── + +describe('sanitizeQuery — strips gold', () => { + test('output has public fields only', () => { + const q = makeQuery(); + const sanitized = sanitizeQuery(q); + expect(sanitized.id).toBe(q.id); + expect(sanitized.tier).toBe(q.tier); + expect(sanitized.text).toBe(q.text); + expect(sanitized.expected_output_type).toBe(q.expected_output_type); + }); + + test('output does NOT have gold', () => { + const q = makeQuery({ gold: { relevant: ['people/amara'], grades: { 'people/amara': 3 } } }); + const sanitized = sanitizeQuery(q); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + expect((sanitized as any).gold).toBeUndefined(); + expect('gold' in sanitized).toBe(false); + }); + + test('retains optional fields (as_of_date, tags, author)', () => { + const q = makeQuery({ + as_of_date: '2026-04-20', + tags: ['temporal'], + author: 'internal', + acceptable_variants: ['who works at Halfway'], + known_failure_modes: ['bare-name-collision'], + }); + const sanitized = sanitizeQuery(q); + expect(sanitized.as_of_date).toBe('2026-04-20'); + expect(sanitized.tags).toEqual(['temporal']); + expect(sanitized.author).toBe('internal'); + expect(sanitized.acceptable_variants).toEqual(['who works at Halfway']); + expect(sanitized.known_failure_modes).toEqual(['bare-name-collision']); + }); + + test('omits undefined optional fields from the sanitized shape', () => { + const q = makeQuery(); // no as_of_date, no tags, etc. + const sanitized = sanitizeQuery(q); + expect('as_of_date' in sanitized).toBe(false); + expect('tags' in sanitized).toBe(false); + }); + + test('output is a NEW object', () => { + const q = makeQuery(); + expect(sanitizeQuery(q)).not.toBe(q as unknown as PublicQuery); + }); +}); + +// ─── Proxy-based adversarial adapter simulation ─────────────────────── + +describe('adversarial adapter access — Proxy tripwire', () => { + test('Proxy-wrapped PublicPage throws on `_facts` access (tripwire)', () => { + const sanitized = sanitizePage(makeRichPage({ _facts: { type: 'person' } })); + const tripwire = new Proxy(sanitized, { + get(target, prop) { + if (prop === '_facts' || prop === 'gold') { + throw new Error(`sealed-qrels violation: adapter read forbidden field "${String(prop)}"`); + } + return target[prop as keyof PublicPage]; + }, + }); + // Normal reads work + expect(tripwire.slug).toBe('people/amara'); + // Adversarial read throws + // eslint-disable-next-line @typescript-eslint/no-explicit-any + expect(() => (tripwire as any)._facts).toThrow(/sealed-qrels violation/); + }); + + test('Proxy-wrapped PublicQuery throws on `gold` access', () => { + const sanitized = sanitizeQuery(makeQuery()); + const tripwire = new Proxy(sanitized, { + get(target, prop) { + if (prop === 'gold') { + throw new Error('sealed-qrels violation: adapter read q.gold'); + } + return target[prop as keyof PublicQuery]; + }, + }); + expect(tripwire.id).toBe('q-0001'); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + expect(() => (tripwire as any).gold).toThrow(/sealed-qrels violation/); + }); +}); + +// ─── Honest documentation of the seal's limits ──────────────────────── + +describe('soft-seal documentation', () => { + test('sanitize cannot protect against filesystem access', () => { + // This test is intentionally EDUCATIONAL — it documents that the seal + // is only at the object level. A malicious adapter that does + // readFileSync('eval/data/gold/qrels.json') bypasses the seal entirely. + // BrainBench v2's Docker sandbox is the real enforcement. + const pseudocode = + "const gold = JSON.parse(readFileSync('eval/data/gold/qrels.json'))"; + expect(pseudocode.length).toBeGreaterThan(0); + // The defense is deliberate and documented in types.ts. + }); + + test('sanitize cannot prevent a malicious Proxy setup from the adapter', () => { + // Similarly: a malicious adapter could set up its own Proxy to probe + // values. The seal is "can a well-behaved adapter accidentally cheat?", + // not "can a malicious adapter never cheat?" + const sanitized = sanitizePage(makeRichPage()); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const asAny = sanitized as any; + expect(asAny._facts).toBeUndefined(); + // A malicious adapter with network access could still exfiltrate the + // page list and correlate externally. Hard enforcement requires + // process isolation. + }); +}); + +// ─── Integration: scorer still sees full Query ──────────────────────── + +describe('scorer retains gold', () => { + test('original Query object still has gold after sanitization (immutable copy)', () => { + const q = makeQuery({ gold: { relevant: ['people/amara', 'companies/halfway'] } }); + const sanitized = sanitizeQuery(q); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + expect((sanitized as any).gold).toBeUndefined(); + // Scorer still has access to q.gold.relevant + expect(q.gold.relevant).toEqual(['people/amara', 'companies/halfway']); + }); + + test('original RichPage still has _facts after sanitization', () => { + const rp = makeRichPage({ _facts: { type: 'meeting', attendees: ['people/amara'] } }); + sanitizePage(rp); + expect(rp._facts.attendees).toEqual(['people/amara']); + }); +});