From e2a5dc4a2bcbce90ed58d404f8c8454aa382a5f8 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sun, 19 Apr 2026 00:10:15 +0800 Subject: [PATCH] feat(eval): Phase 2 query validator + Tier 5 Fuzzy + Tier 5.5 synthetic + N=5 tolerance bands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes multiple Phase 2 items in one commit since they form a cohesive package: query schema enforcement + new query tiers + per-query-set statistical rigor. Added: eval/runner/queries/validator.ts — hand-rolled Query schema validator eval/runner/queries/validator.test.ts — 24 unit tests, all pass eval/runner/queries/tier5-fuzzy.ts — 30 hand-authored Tier 5 Fuzzy/Vibe queries eval/runner/queries/tier5_5-synthetic.ts — 50 SYNTHETIC-labeled outsider-style queries (author: "synthetic-outsider-v1") eval/runner/queries/index.ts — aggregator + validateAll() Modified: eval/runner/multi-adapter.ts — N=5 runs per adapter (BRAINBENCH_N override), page-order shuffle, mean±stddev reporting Query validator (hand-rolled, no zod dep to match gbrain codebase style): - Temporal verb regex enforces as_of_date (per eng pass 2 spec): /\\b(is|was|were|current|now|at the time|during|as of|when did)\\b/i - Validates tier enum, expected_output_type enum, gold shape per type - gold.relevant must be non-empty slug[] for cited-source-pages queries - abstention requires gold.expected_abstention === true - externally-authored tier requires author field - batch validation catches duplicate IDs Tier 5 Fuzzy/Vibe (30 queries, hand-authored): - Vague recall: "Someone who was a senior engineer at a biotech company..." - Trait-based: "The engineer who pushed back on microservices" - Cultural/epithet: "Who is known as a 'systems builder' in security?" - Abstention bait: "Which Layer 1 project did the crypto guy leave?" (prose mentions but never names; good systems abstain) - Addresses Codex's circularity critique — vague queries where graph-heavy systems shouldn't inherently win. Tier 5.5 Synthetic Outsider (50 queries, AI-authored placeholder): - Clearly labeled author: "synthetic-outsider-v1" - Phrasing variety not in the 4 template families: * fragment style ("crypto founder Goldman Sachs background") * polite/natural ("Can you pull up what we have on...") * comparison ("What is the difference between X and Y?") * follow-up ("And who else advises Orbit Labs?") * typos/misspellings ("adam lopez bioinformatcis") * similarity ("Find me someone like Alice Davis...") * imperative ("Pull up Alice Davis") - Real Tier 5.5 from outside researchers supersedes synthetic via PRs to eval/external-authors/ (docs ship in follow-up commit). N=5 tolerance bands: - Default N=5, override via BRAINBENCH_N env var (e.g. BRAINBENCH_N=1 for dev loops) - Per-run seeded Fisher-Yates shuffle of page ingest order (LCG seed = run_idx+1) - Surfaces order-dependent adapter bugs (tie-break-by-first-seen etc.) - Reports mean ± sample-stddev per metric - "stddev = 0" is honest signal that the adapter is deterministic, not a bug. LLM-judge metrics (future) will naturally produce non-zero stddev. Validation: all 80 Tier 5 + 5.5 queries pass validateAll(). 24 validator unit tests pass. Next commit: world.html contributor explorer (Phase 3). Co-Authored-By: Claude Opus 4.7 (1M context) --- eval/runner/multi-adapter.ts | 118 +++++++- eval/runner/queries/index.ts | 38 +++ eval/runner/queries/tier5-fuzzy.ts | 308 ++++++++++++++++++++ eval/runner/queries/tier5_5-synthetic.ts | 346 +++++++++++++++++++++++ eval/runner/queries/validator.test.ts | 209 ++++++++++++++ eval/runner/queries/validator.ts | 204 +++++++++++++ 6 files changed, 1208 insertions(+), 15 deletions(-) create mode 100644 eval/runner/queries/index.ts create mode 100644 eval/runner/queries/tier5-fuzzy.ts create mode 100644 eval/runner/queries/tier5_5-synthetic.ts create mode 100644 eval/runner/queries/validator.test.ts create mode 100644 eval/runner/queries/validator.ts diff --git a/eval/runner/multi-adapter.ts b/eval/runner/multi-adapter.ts index 8e966c028..8247614ee 100644 --- a/eval/runner/multi-adapter.ts +++ b/eval/runner/multi-adapter.ts @@ -263,22 +263,69 @@ function parseRelationalQuery( return { seed: '', direction: 'in', linkTypes: [] }; } -// ─── Scorecard ────────────────────────────────────────────────────── +// ─── Tolerance bands (N-run variance measurement) ────────────────── -interface AdapterScorecard { - adapter: string; - queries: number; +/** + * N=5 per eng pass 3 decision. For current adapters (all deterministic + * over sorted page input) bands will be ~0. Per-run variance surfaces + * when any of these enter the benchmark: + * - LLM-judge scoring (future) + * - Non-deterministic embedding providers + * - Page-ordering-dependent dedup tie-breaks (induced here by shuffle) + * + * Shuffling ingestion order per run reveals order-sensitive bugs. An + * adapter with hidden order-dependence (e.g. a tie-break that favors + * first-seen slug) shows up as non-zero stddev. + */ +const RUNS_PER_ADAPTER = Number(process.env.BRAINBENCH_N ?? '5'); + +interface RunResult { mean_precision_at_k: number; mean_recall_at_k: number; correct_in_top_k: number; total_expected: number; } -async function scoreAdapter( +interface AdapterScorecard { + adapter: string; + queries: number; + runs: number; + /** Mean across N runs. */ + mean_precision_at_k: number; + mean_recall_at_k: number; + /** Sample stddev across N runs (n-1 denominator). Zero means deterministic. */ + stddev_precision_at_k: number; + stddev_recall_at_k: number; + /** From the first run (for the headline "correct/gold" column). */ + correct_in_top_k: number; + total_expected: number; +} + +/** + * Seeded Fisher-Yates shuffle. Deterministic given the same seed so + * N-run results are reproducible by anyone re-running with the same seed. + * Uses a linear congruential generator (LCG) — good enough for benchmark + * permutations, not cryptographic. + */ +function shuffleSeeded(arr: T[], seed: number): T[] { + const out = [...arr]; + let s = seed >>> 0; + const next = () => { + s = (s * 1664525 + 1013904223) >>> 0; + return s / 0x100000000; + }; + for (let i = out.length - 1; i > 0; i--) { + const j = Math.floor(next() * (i + 1)); + [out[i], out[j]] = [out[j], out[i]]; + } + return out; +} + +async function scoreOneRun( adapter: Adapter, pages: Page[], queries: Query[], -): Promise { +): Promise { const state = await adapter.init(pages, { name: adapter.name }); let totalP = 0; let totalR = 0; @@ -289,14 +336,11 @@ async function scoreAdapter( const relevant = new Set(q.gold.relevant ?? []); totalP += precisionAtK(results, relevant, TOP_K); totalR += recallAtK(results, relevant, TOP_K); - // Exact top-K correct count for the headline table. const topK = results.slice(0, TOP_K); for (const r of topK) if (relevant.has(r.page_id)) totalCorrect++; totalExpected += relevant.size; } return { - adapter: adapter.name, - queries: queries.length, mean_precision_at_k: queries.length > 0 ? totalP / queries.length : 0, mean_recall_at_k: queries.length > 0 ? totalR / queries.length : 0, correct_in_top_k: totalCorrect, @@ -304,10 +348,53 @@ async function scoreAdapter( }; } +function stddev(values: number[]): number { + const n = values.length; + if (n < 2) return 0; + const mean = values.reduce((a, b) => a + b, 0) / n; + const variance = values.reduce((a, b) => a + (b - mean) ** 2, 0) / (n - 1); + return Math.sqrt(variance); +} + +async function scoreAdapter( + adapter: Adapter, + pages: Page[], + queries: Query[], +): Promise { + const runResults: RunResult[] = []; + for (let i = 0; i < RUNS_PER_ADAPTER; i++) { + // Shuffle pages per run with a per-run seed. Seed = i + 1 (not 0, + // since LCG iterates once at start of next()). Run 0 uses the seed + // that produces a minimally-scrambled permutation; doesn't matter + // for correctness since we aggregate across runs. + const shuffled = shuffleSeeded(pages, i + 1); + const r = await scoreOneRun(adapter, shuffled, queries); + runResults.push(r); + } + const pVals = runResults.map(r => r.mean_precision_at_k); + const rVals = runResults.map(r => r.mean_recall_at_k); + return { + adapter: adapter.name, + queries: queries.length, + runs: RUNS_PER_ADAPTER, + mean_precision_at_k: pVals.reduce((a, b) => a + b, 0) / pVals.length, + mean_recall_at_k: rVals.reduce((a, b) => a + b, 0) / rVals.length, + stddev_precision_at_k: stddev(pVals), + stddev_recall_at_k: stddev(rVals), + correct_in_top_k: runResults[0].correct_in_top_k, + total_expected: runResults[0].total_expected, + }; +} + function pct(n: number, digits = 1): string { return `${(n * 100).toFixed(digits)}%`; } +function pctBand(mean: number, sd: number, digits = 1): string { + if (sd === 0) return pct(mean, digits); + return `${pct(mean, digits)} \u00b1${(sd * 100).toFixed(digits)}`; +} + // ─── Main ────────────────────────────────────────────────────────── async function main() { @@ -336,24 +423,25 @@ async function main() { process.exit(1); } - log('## Running adapters\n'); + log(`## Running adapters (N=${RUNS_PER_ADAPTER} runs per adapter, page-order shuffled per run)\n`); const scorecards: AdapterScorecard[] = []; for (const a of adapters) { log(`- ${a.name} ...`); const t0 = Date.now(); const sc = await scoreAdapter(a, pages, queries); const elapsed = ((Date.now() - t0) / 1000).toFixed(1); - log(` done (${elapsed}s). P@${TOP_K} ${pct(sc.mean_precision_at_k)}, R@${TOP_K} ${pct(sc.mean_recall_at_k)}, ${sc.correct_in_top_k}/${sc.total_expected} correct in top-${TOP_K}`); + log(` done (${elapsed}s). P@${TOP_K} ${pctBand(sc.mean_precision_at_k, sc.stddev_precision_at_k)}, R@${TOP_K} ${pctBand(sc.mean_recall_at_k, sc.stddev_recall_at_k)}, ${sc.correct_in_top_k}/${sc.total_expected} correct (run 1)`); scorecards.push(sc); } - log('\n## Side-by-side scorecard\n'); - log(`| Adapter | Queries | P@${TOP_K} | R@${TOP_K} | Correct in top-${TOP_K} | Gold total |`); - log('|---------------------|---------|--------|--------|---------------------|------------|'); + log('\n## Side-by-side scorecard (mean \u00b1 stddev across N runs)\n'); + log(`| Adapter | Runs | Queries | P@${TOP_K} (mean \u00b1 sd) | R@${TOP_K} (mean \u00b1 sd) |`); + log('|---------------------|------|---------|---------------------|---------------------|'); for (const sc of scorecards) { - log(`| ${sc.adapter.padEnd(19)} | ${String(sc.queries).padStart(7)} | ${pct(sc.mean_precision_at_k).padStart(6)} | ${pct(sc.mean_recall_at_k).padStart(6)} | ${String(sc.correct_in_top_k).padStart(19)} | ${String(sc.total_expected).padStart(10)} |`); + log(`| ${sc.adapter.padEnd(19)} | ${String(sc.runs).padStart(4)} | ${String(sc.queries).padStart(7)} | ${pctBand(sc.mean_precision_at_k, sc.stddev_precision_at_k).padStart(19)} | ${pctBand(sc.mean_recall_at_k, sc.stddev_recall_at_k).padStart(19)} |`); } log(''); + log('*Stddev = 0 means the adapter is deterministic over page ordering. Non-zero stddev surfaces order-dependent bugs (e.g. tie-break that favors first-seen slug). LLM-judge-based metrics will produce non-zero stddev once added.*\n'); if (scorecards.length >= 2) { const [first, ...rest] = scorecards; diff --git a/eval/runner/queries/index.ts b/eval/runner/queries/index.ts new file mode 100644 index 000000000..af558c21a --- /dev/null +++ b/eval/runner/queries/index.ts @@ -0,0 +1,38 @@ +/** + * Aggregates all tier-5/5.5 query sets and exposes validator helpers. + * + * Usage: + * import { getAllTierQueries, validateAll } from './queries'; + * const queries = getAllTierQueries(); + * const result = validateAll(queries); + */ + +import type { Query } from '../types.ts'; +import { getTier5FuzzyQueries } from './tier5-fuzzy.ts'; +import { getTier5_5SyntheticQueries } from './tier5_5-synthetic.ts'; +import { validateQuerySet, formatIssues } from './validator.ts'; + +/** Tier 5 Fuzzy/Vibe (hand-authored by gstack maintainers). */ +export { getTier5FuzzyQueries } from './tier5-fuzzy.ts'; + +/** Tier 5.5 externally-authored (SYNTHETIC placeholder; see CONTRIBUTING.md). */ +export { getTier5_5SyntheticQueries } from './tier5_5-synthetic.ts'; + +export { validateQuery, validateQuerySet, formatIssues, TEMPORAL_VERBS } from './validator.ts'; +export type { ValidationIssue, ValidationResult } from './validator.ts'; + +/** All Tier 5 + 5.5 queries concatenated. */ +export function getAllTierQueries(): Query[] { + return [...getTier5FuzzyQueries(), ...getTier5_5SyntheticQueries()]; +} + +/** Validate the complete Tier-5 + 5.5 set (used by CI + eval:query:validate). */ +export function validateAll(): { ok: boolean; count: number; report: string } { + const queries = getAllTierQueries(); + const result = validateQuerySet(queries); + return { + ok: result.ok, + count: queries.length, + report: formatIssues(result), + }; +} diff --git a/eval/runner/queries/tier5-fuzzy.ts b/eval/runner/queries/tier5-fuzzy.ts new file mode 100644 index 000000000..ebe3dc9fe --- /dev/null +++ b/eval/runner/queries/tier5-fuzzy.ts @@ -0,0 +1,308 @@ +/** + * Tier 5: Fuzzy / Vibe queries. + * + * Vague recall and "I know I mentioned this somewhere" — the kind of query + * real people ask their brain when they can't quite remember the exact + * entity name. Graph-heavy systems shouldn't have an inherent edge here + * because the query doesn't mention the target entity precisely. + * + * Per the 4-review arc: these address Codex's circularity critique + * ("gbrain's adversarial list is its product roadmap"). If gbrain loses + * ground on vague queries while winning on relational ones, that's an + * honest tradeoff story. If gbrain wins on BOTH, that's a stronger + * benchmark claim. + * + * Target: ~30 queries (statistical floor per eng pass 3). + * + * Gold derivation: each query specifies an expected answer slug set based + * on the canonical world (eval/data/world-v1/_ledger.json). We fix a small + * set of landmarks here rather than deriving from _facts — fuzzy queries + * don't map 1:1 to _facts fields. + */ + +import type { Query } from '../types.ts'; + +/** + * Hand-authored Tier 5 vibe queries. Each targets real entities from + * eval/data/world-v1/ that actually exist in the corpus. Slugs verified + * against world-v1 shards at authoring time. + */ +export const TIER5_FUZZY_QUERIES: Query[] = [ + // ── "I know I mentioned this somewhere" style ───────────────────── + { + id: 'q5-0001', + tier: 'fuzzy', + text: 'Someone I know was a senior engineer at a biotech company doing drug discovery — who?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lopez-113'] }, + as_of_date: 'per-source', + tags: ['vague-recall', 'role-based'], + known_failure_modes: ['might return all biotech company pages; we want the person'], + }, + { + id: 'q5-0002', + tier: 'fuzzy', + text: 'The crypto-infra founder who did a stint at Goldman before building his own thing', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lee-19'] }, + tags: ['vague-recall', 'biographical'], + }, + { + id: 'q5-0003', + tier: 'fuzzy', + text: 'The security advisor woman based in Boston, multi-year engagement with a cybersecurity startup', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/alice-davis-172'] }, + tags: ['vague-recall', 'role-based', 'location'], + }, + + // ── Summarization-over-messy-notes style ────────────────────────── + { + id: 'q5-0004', + tier: 'fuzzy', + text: 'Summarize what we know about founders who raised Series A in 2024.', + expected_output_type: 'answer-string', + gold: { relevant: [] }, + acceptable_variants: ['Series A 2024 founders summary'], + tags: ['summarization', 'multi-entity'], + known_failure_modes: ['accept any top-K that includes actual Series-A-2024 founders'], + }, + { + id: 'q5-0005', + tier: 'fuzzy', + text: 'Who are the fintech advisors in our network?', + expected_output_type: 'abstention', + gold: { expected_abstention: true }, + tags: ['summarization', 'role-intersection', 'partial-in-corpus'], + known_failure_modes: ['"fintech" advisors are scarce in twin-amara corpus; a good system abstains or flags partial match'], + }, + { + id: 'q5-0006', + tier: 'fuzzy', + text: 'Tell me about the people who push back hard on microservices', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lopez-113'] }, + tags: ['trait-based', 'opinion'], + known_failure_modes: ['requires catching "controversial internal memo on microservices" in prose'], + }, + + // ── Partial-information recall ──────────────────────────────────── + { + id: 'q5-0007', + tier: 'fuzzy', + text: 'Who has a "40 under 40" mention?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lee-19'] }, + tags: ['biographical-fragment'], + }, + { + id: 'q5-0008', + tier: 'fuzzy', + text: 'The company whose CEO insists on founder-friendly terms and minimal board seats', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['companies/forge-19'] }, + tags: ['trait-based', 'culture'], + }, + { + id: 'q5-0009', + tier: 'fuzzy', + text: 'Someone we know who cut sequencing pipeline runtime by about 40%', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lopez-113'] }, + tags: ['achievement-based'], + }, + { + id: 'q5-0010', + tier: 'fuzzy', + text: 'The person who wrote an internal memo about deleting half the microservices', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lopez-113'] }, + tags: ['behavioral-recall'], + }, + + // ── "what was that thing about..." style ────────────────────────── + { + id: 'q5-0011', + tier: 'fuzzy', + text: 'What was the thing about MEV-resistant transaction ordering?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lee-19'] }, + as_of_date: 'per-source', + tags: ['topic-recall'], + }, + { + id: 'q5-0012', + tier: 'fuzzy', + text: 'The partner who focuses on early-stage fintech', + expected_output_type: 'abstention', + gold: { expected_abstention: true }, + tags: ['role-based', 'domain', 'partial-in-corpus'], + known_failure_modes: ['multiple partial matches; good systems either abstain or flag "multiple candidates"'], + }, + { + id: 'q5-0013', + tier: 'fuzzy', + text: 'Which Layer 1 project did that crypto guy leave over tokenomics disagreements?', + expected_output_type: 'abstention', + gold: { expected_abstention: true }, + known_failure_modes: ['prose says a Layer 1 project but never names it; good systems abstain'], + tags: ['abstention', 'under-specified'], + }, + { + id: 'q5-0014', + tier: 'fuzzy', + text: 'Who built that cross-chain messaging protocol?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['companies/forge-19'] }, + tags: ['product-feature-recall'], + }, + { + id: 'q5-0015', + tier: 'fuzzy', + text: 'The engineer who is notoriously demanding on code review', + expected_output_type: 'abstention', + gold: { expected_abstention: true }, + as_of_date: 'corpus-end', + known_failure_modes: ['no single canonical answer in corpus; good systems flag ambiguity'], + tags: ['trait-based', 'partial-in-corpus'], + }, + + // ── Emotional / cultural recall ────────────────────────────────── + { + id: 'q5-0016', + tier: 'fuzzy', + text: 'The company whose culture is described as "either loved or hated"', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['companies/forge-19'] }, + as_of_date: 'per-source', + tags: ['culture-recall'], + }, + { + id: 'q5-0017', + tier: 'fuzzy', + text: 'The advisor who pushed hard for zero-trust architecture overhaul', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/alice-davis-172'] }, + tags: ['behavior-recall', 'advocacy'], + }, + { + id: 'q5-0018', + tier: 'fuzzy', + text: 'Someone who speaks selectively at conferences and prefers small venues', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/alice-davis-172'] }, + tags: ['preference-recall'], + }, + { + id: 'q5-0019', + tier: 'fuzzy', + text: 'Who is rumored to be writing a book on security culture?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/alice-davis-172'] }, + as_of_date: 'corpus-end', + tags: ['gossip', 'side-project'], + }, + + // ── "something about X" generic-topic style ────────────────────── + { + id: 'q5-0020', + tier: 'fuzzy', + text: 'Any portfolio companies focused on drug discovery?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['companies/delta-3'] }, + tags: ['topical', 'industry'], + }, + { + id: 'q5-0021', + tier: 'fuzzy', + text: 'Who among our founders worked at Goldman Sachs?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lee-19'] }, + tags: ['background-intersection'], + }, + { + id: 'q5-0022', + tier: 'fuzzy', + text: 'The person with an MIT CS background who dropped out of a PhD', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lee-19'] }, + tags: ['biography-fragment'], + }, + { + id: 'q5-0023', + tier: 'fuzzy', + text: 'Who among our people is a long-distance runner?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lopez-113'] }, + as_of_date: 'corpus-end', + tags: ['personal-trait'], + }, + + // ── Negative / abstention fuzzy (known failure bait) ───────────── + { + id: 'q5-0024', + tier: 'fuzzy', + text: 'Which YC W18 founder built an analytics dashboard?', + expected_output_type: 'abstention', + gold: { expected_abstention: true }, + tags: ['abstention', 'not-in-corpus'], + known_failure_modes: ['W18 batch doesn\'t exist in this corpus; good systems abstain'], + }, + { + id: 'q5-0025', + tier: 'fuzzy', + text: 'Who founded the developer-tools company that got acquihired by Roche?', + expected_output_type: 'abstention', + gold: { expected_abstention: true }, + tags: ['abstention', 'mentioned-but-not-named'], + known_failure_modes: ['prose mentions a bioinformatics startup acquired by Roche but never names it; good systems abstain'], + }, + + // ── Cross-referencing without exact entity names ───────────────── + { + id: 'q5-0026', + tier: 'fuzzy', + text: 'What companies have enterprise security architecture help from an outside advisor?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['companies/prism-43'] }, + tags: ['relational-fuzzy'], + }, + { + id: 'q5-0027', + tier: 'fuzzy', + text: 'Who is known as a "systems builder" in the security space?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/alice-davis-172'] }, + as_of_date: 'corpus-end', + tags: ['epithet-recall'], + }, + { + id: 'q5-0028', + tier: 'fuzzy', + text: 'The Boston-based person who completed a SOC 2 audit for someone', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/alice-davis-172'] }, + tags: ['achievement-plus-location'], + }, + { + id: 'q5-0029', + tier: 'fuzzy', + text: 'Which advisor has SecureCon Northeast speaking experience?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/alice-davis-172'] }, + tags: ['conference-history'], + }, + { + id: 'q5-0030', + tier: 'fuzzy', + text: 'Someone who published multiple technical papers on cryptographic primitives', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lee-19'] }, + tags: ['academic-output'], + }, +]; + +export function getTier5FuzzyQueries(): Query[] { + // Defensive copy so callers can't mutate the canonical set. + return TIER5_FUZZY_QUERIES.map(q => ({ ...q, gold: { ...q.gold }, tags: q.tags ? [...q.tags] : undefined })); +} diff --git a/eval/runner/queries/tier5_5-synthetic.ts b/eval/runner/queries/tier5_5-synthetic.ts new file mode 100644 index 000000000..b2aeff40c --- /dev/null +++ b/eval/runner/queries/tier5_5-synthetic.ts @@ -0,0 +1,346 @@ +/** + * Tier 5.5: Externally-Authored queries (SYNTHETIC placeholder). + * + * Per plan file: real Tier 5.5 requires 2-3 outside researchers writing + * ~50 queries each against the committed corpus. That's a human-in-the-loop + * deliverable (see eval/CONTRIBUTING.md for how to submit). + * + * This file ships a SYNTHETIC placeholder set: AI-authored queries that + * deliberately vary phrasing patterns to simulate what an outsider would + * write. They are CLEARLY labeled via `author: "synthetic-outsider-v1"` + * so real researcher submissions can supersede them without ambiguity. + * + * Why ship synthetic ones at all: + * - Tier 5.5 has to exist in the scorecard for the multi-axis report + * to have a full column. A missing tier reads as "not measured." + * - The synthetic set establishes phrasing variety (full sentences, + * short fragments, follow-up style, comparison style, "what's the + * difference between X and Y" style) that the 4 template families + * in the medium tier don't cover. + * - When real researchers submit, scorecards can compare "AI-authored" + * vs "human-authored" columns to flag where LLM judgment differs + * from human judgment on the same corpus. + * + * Author field: "synthetic-outsider-v1" for every query here. + * + * Gold verification: slugs referenced below exist in eval/data/world-v1/. + * Small slug set for authenticity; entity pages cited: adam-lee-19, + * adam-lopez-113, alice-davis-172, forge-19, delta-3, prism-43, + * orbit-labs-92 (+ others from the corpus). + */ + +import type { Query } from '../types.ts'; + +const AUTHOR = 'synthetic-outsider-v1'; + +export const TIER5_5_SYNTHETIC_QUERIES: Query[] = [ + // ─── Short-fragment style (how real researchers write notes) ───── + { id: 'q55-0001', tier: 'externally-authored', author: AUTHOR, + text: 'crypto founder Goldman Sachs background', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lee-19'] }, + tags: ['fragment-style'] }, + { id: 'q55-0002', tier: 'externally-authored', author: AUTHOR, + text: 'Prism cybersecurity advisor engagement history', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/alice-davis-172', 'companies/prism-43'] }, + tags: ['fragment-style', 'relational'] }, + { id: 'q55-0003', tier: 'externally-authored', author: AUTHOR, + text: 'Delta biotech engineers', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lopez-113'] }, + tags: ['fragment-style'] }, + { id: 'q55-0004', tier: 'externally-authored', author: AUTHOR, + text: 'Forge crypto infrastructure founder details', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lee-19', 'companies/forge-19'] }, + tags: ['fragment-style', 'founder-company'] }, + + // ─── Full-sentence style with natural hedging ───────────────────── + { id: 'q55-0005', tier: 'externally-authored', author: AUTHOR, + text: 'Can you pull up what we have on the founder who left a Layer 1 project over tokenomics?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lee-19'] }, + tags: ['polite-natural'] }, + { id: 'q55-0006', tier: 'externally-authored', author: AUTHOR, + text: 'I need the background on the Delta senior engineer who cut pipeline runtime', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lopez-113'] }, + tags: ['polite-natural', 'specific-achievement'] }, + { id: 'q55-0007', tier: 'externally-authored', author: AUTHOR, + text: 'Please find the advisor with SOC 2 audit prep experience at Orbit Labs', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/alice-davis-172'] }, + tags: ['polite-natural', 'relational'] }, + + // ─── Comparison / differentiation style ─────────────────────────── + { id: 'q55-0008', tier: 'externally-authored', author: AUTHOR, + text: 'What is the difference between Adam Lee and Adam Lopez in our network?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lee-19', 'people/adam-lopez-113'] }, + as_of_date: 'corpus-end', + tags: ['disambiguation', 'comparison'] }, + { id: 'q55-0009', tier: 'externally-authored', author: AUTHOR, + text: 'Compare Forge and Delta as companies in our portfolio', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['companies/forge-19', 'companies/delta-3'] }, + tags: ['comparison'] }, + { id: 'q55-0010', tier: 'externally-authored', author: AUTHOR, + text: 'Which of our advisors is based on the East Coast?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/alice-davis-172'] }, + as_of_date: 'corpus-end', + tags: ['location-filter'] }, + + // ─── Who-does-what style (role-first) ────────────────────────────── + { id: 'q55-0011', tier: 'externally-authored', author: AUTHOR, + text: 'Who focuses on synthetic biology at Delta?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lopez-113'] }, + tags: ['role-first'] }, + { id: 'q55-0012', tier: 'externally-authored', author: AUTHOR, + text: 'Who wrote the whitepaper on MEV-resistant transaction ordering?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lee-19'] }, + tags: ['achievement-first'] }, + { id: 'q55-0013', tier: 'externally-authored', author: AUTHOR, + text: 'Who would you ask about enterprise security architecture?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/alice-davis-172'] }, + tags: ['skill-lookup'] }, + { id: 'q55-0014', tier: 'externally-authored', author: AUTHOR, + text: 'Who is the expert on bioinformatics pipelines in our network?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lopez-113'] }, + as_of_date: 'corpus-end', + tags: ['expertise-lookup'] }, + + // ─── Follow-up style (assumes prior context) ────────────────────── + { id: 'q55-0015', tier: 'externally-authored', author: AUTHOR, + text: 'And who else advises Orbit Labs?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/alice-davis-172'] }, + tags: ['follow-up', 'assumed-context'] }, + { id: 'q55-0016', tier: 'externally-authored', author: AUTHOR, + text: 'Also at Delta?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lopez-113'] }, + tags: ['follow-up', 'minimal'] }, + + // ─── Characteristic / trait recall ───────────────────────────────── + { id: 'q55-0017', tier: 'externally-authored', author: AUTHOR, + text: 'Our demanding engineering leader with the long-term vision approach', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lee-19'] }, + tags: ['trait-description'] }, + { id: 'q55-0018', tier: 'externally-authored', author: AUTHOR, + text: 'The fast-shipping opinionated engineer (likes Postgres, hates meetings)', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lopez-113'] }, + tags: ['multi-trait'] }, + { id: 'q55-0019', tier: 'externally-authored', author: AUTHOR, + text: 'A systems-builder style advisor who scales security architecture', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/alice-davis-172'] }, + tags: ['epithet-plus-role'] }, + + // ─── Misspellings / typos (real researchers make these) ─────────── + { id: 'q55-0020', tier: 'externally-authored', author: AUTHOR, + text: 'adam lee the crypto guy', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lee-19'] }, + tags: ['lowercase', 'minimal-name'] }, + { id: 'q55-0021', tier: 'externally-authored', author: AUTHOR, + text: 'alice davis cybersecuirty advisor', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/alice-davis-172'] }, + tags: ['typo', 'lowercase'] }, + { id: 'q55-0022', tier: 'externally-authored', author: AUTHOR, + text: 'adam lopez bioinformatcis', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lopez-113'] }, + tags: ['typo'] }, + + // ─── "Find me someone like..." semantic-similarity style ───────── + { id: 'q55-0023', tier: 'externally-authored', author: AUTHOR, + text: 'Find me someone like Alice Davis for our new security engagement', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/alice-davis-172'] }, + tags: ['recommendation', 'similarity'] }, + { id: 'q55-0024', tier: 'externally-authored', author: AUTHOR, + text: 'Who has a profile similar to Adam Lopez — fast shipper, infrastructure background?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lopez-113'] }, + tags: ['similarity', 'profile-match'] }, + + // ─── Negative / "is there anyone..." phrasing ──────────────────── + { id: 'q55-0025', tier: 'externally-authored', author: AUTHOR, + text: 'Is there anyone in our network who has given a Mainnet conference keynote?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lee-19'] }, + as_of_date: 'corpus-end', + tags: ['existence-check'] }, + { id: 'q55-0026', tier: 'externally-authored', author: AUTHOR, + text: 'Does anyone know someone who has published on cryptographic primitives?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lee-19'] }, + tags: ['existence-check', 'topic-lookup'] }, + + // ─── "What does X do?" direct-entity-name style ────────────────── + { id: 'q55-0027', tier: 'externally-authored', author: AUTHOR, + text: 'What does Forge do?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['companies/forge-19'] }, + tags: ['direct-lookup'] }, + { id: 'q55-0028', tier: 'externally-authored', author: AUTHOR, + text: 'What is Prism working on these days?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['companies/prism-43'] }, + as_of_date: 'corpus-end', + tags: ['direct-lookup', 'temporal-latest'] }, + { id: 'q55-0029', tier: 'externally-authored', author: AUTHOR, + text: "Tell me about Delta's drug discovery platform", + expected_output_type: 'cited-source-pages', + gold: { relevant: ['companies/delta-3'] }, + tags: ['direct-lookup', 'domain-specific'] }, + + // ─── Cross-cutting relationship queries ───────────────────────── + { id: 'q55-0030', tier: 'externally-authored', author: AUTHOR, + text: 'People we know who are associated with both biotech and software infrastructure', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lopez-113'] }, + tags: ['cross-domain'] }, + { id: 'q55-0031', tier: 'externally-authored', author: AUTHOR, + text: 'Companies where our advisors have multi-year ongoing relationships', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['companies/prism-43'] }, + tags: ['relationship-depth'] }, + + // ─── Role-plus-industry intersection ────────────────────────────── + { id: 'q55-0032', tier: 'externally-authored', author: AUTHOR, + text: 'Any security-focused advisors for enterprise clients?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/alice-davis-172'] }, + tags: ['role-industry-intersect'] }, + { id: 'q55-0033', tier: 'externally-authored', author: AUTHOR, + text: 'Senior infrastructure engineers in synthetic biology', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lopez-113'] }, + tags: ['role-industry-intersect'] }, + + // ─── "Pull up..." imperative style ─────────────────────────────── + { id: 'q55-0034', tier: 'externally-authored', author: AUTHOR, + text: 'Pull up Alice Davis', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/alice-davis-172'] }, + tags: ['imperative', 'direct-name'] }, + { id: 'q55-0035', tier: 'externally-authored', author: AUTHOR, + text: 'Show me the Forge page', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['companies/forge-19'] }, + tags: ['imperative', 'direct-page'] }, + + // ─── Background-check style ────────────────────────────────────── + { id: 'q55-0036', tier: 'externally-authored', author: AUTHOR, + text: 'Prior experience of Delta senior engineers before joining', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lopez-113'] }, + tags: ['background-check'] }, + { id: 'q55-0037', tier: 'externally-authored', author: AUTHOR, + text: 'Educational background of Forge founder', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lee-19'] }, + tags: ['biographical'] }, + + // ─── "When did X..." abstention-adjacent ───────────────────────── + { id: 'q55-0038', tier: 'externally-authored', author: AUTHOR, + text: 'When did Adam Lopez earn his masters degree?', + expected_output_type: 'abstention', + gold: { expected_abstention: true }, + as_of_date: 'corpus-end', + tags: ['abstention', 'not-in-corpus'], + known_failure_modes: ['no mention of Lopez getting a masters; abstain'] }, + { id: 'q55-0039', tier: 'externally-authored', author: AUTHOR, + text: 'Was Alice Davis ever at Palo Alto Networks?', + expected_output_type: 'abstention', + gold: { expected_abstention: true }, + as_of_date: 'corpus-end', + tags: ['abstention', 'speculation-bait'] }, + + // ─── Aggregation queries ───────────────────────────────────────── + { id: 'q55-0040', tier: 'externally-authored', author: AUTHOR, + text: 'List all advisors in our corpus', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/alice-davis-172'] }, + tags: ['aggregation', 'partial-gold'], + known_failure_modes: ['partial gold; accept any adviser-role pages in top-K'] }, + { id: 'q55-0041', tier: 'externally-authored', author: AUTHOR, + text: 'All senior engineers', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lopez-113'] }, + tags: ['aggregation', 'role-filter'] }, + + // ─── Topic-with-no-entity-name (true semantic) ─────────────────── + { id: 'q55-0042', tier: 'externally-authored', author: AUTHOR, + text: 'Zero-trust architecture work', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/alice-davis-172', 'companies/prism-43'] }, + tags: ['topic-only'] }, + { id: 'q55-0043', tier: 'externally-authored', author: AUTHOR, + text: 'Mainnet cross-chain messaging', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['companies/forge-19'] }, + tags: ['topic-only'] }, + { id: 'q55-0044', tier: 'externally-authored', author: AUTHOR, + text: 'Protein modeling integration work', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lopez-113', 'companies/delta-3'] }, + tags: ['topic-only'] }, + + // ─── Natural-language "tell me about..." long form ─────────────── + { id: 'q55-0045', tier: 'externally-authored', author: AUTHOR, + text: 'Can you tell me about the Forge team structure and how they think about hiring?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['companies/forge-19', 'people/adam-lee-19'] }, + tags: ['long-form'] }, + { id: 'q55-0046', tier: 'externally-authored', author: AUTHOR, + text: "I want to understand Alice Davis's advisory approach and her involvement at Prism", + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/alice-davis-172', 'companies/prism-43'] }, + tags: ['long-form', 'multi-entity'] }, + + // ─── Temporal / as-of queries (forces validator to require as_of_date) ─ + { id: 'q55-0047', tier: 'externally-authored', author: AUTHOR, + text: 'Was Alice Davis renewing her advisory contract with Prism recently?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/alice-davis-172'] }, + as_of_date: 'corpus-end', + tags: ['temporal'] }, + { id: 'q55-0048', tier: 'externally-authored', author: AUTHOR, + text: 'When did Forge close their Series A?', + expected_output_type: 'time-qualified-answer', + gold: { expected_answer: '2024-06-19' }, + as_of_date: 'corpus-end', + tags: ['temporal', 'exact-date'] }, + { id: 'q55-0049', tier: 'externally-authored', author: AUTHOR, + text: 'What was Adam Lopez doing before he joined Delta?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lopez-113'] }, + as_of_date: 'per-source', + tags: ['temporal', 'biographical'] }, + + // ─── "Give me a short summary" minimal output ──────────────────── + { id: 'q55-0050', tier: 'externally-authored', author: AUTHOR, + text: 'Short summary of Adam Lee', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/adam-lee-19'] }, + tags: ['summary-request'] }, +]; + +export function getTier5_5SyntheticQueries(): Query[] { + return TIER5_5_SYNTHETIC_QUERIES.map(q => ({ + ...q, + gold: { ...q.gold }, + tags: q.tags ? [...q.tags] : undefined, + })); +} diff --git a/eval/runner/queries/validator.test.ts b/eval/runner/queries/validator.test.ts new file mode 100644 index 000000000..25bbeb4af --- /dev/null +++ b/eval/runner/queries/validator.test.ts @@ -0,0 +1,209 @@ +import { describe, test, expect } from 'bun:test'; +import { validateQuery, validateQuerySet, TEMPORAL_VERBS } from './validator.ts'; +import type { Query } from '../types.ts'; + +function mkValidQuery(overrides: Partial = {}): Query { + return { + id: 'q-0001', + tier: 'easy', + text: 'Who founded Acme?', + expected_output_type: 'cited-source-pages', + gold: { relevant: ['people/alice-chen'] }, + ...overrides, + }; +} + +describe('validateQuery — required fields', () => { + test('valid query passes', () => { + const r = validateQuery(mkValidQuery()); + expect(r.ok).toBe(true); + expect(r.issues.length).toBe(0); + }); + + test('missing id fails', () => { + const r = validateQuery(mkValidQuery({ id: '' })); + expect(r.ok).toBe(false); + expect(r.issues.some(i => i.field === 'id')).toBe(true); + }); + + test('missing text fails', () => { + const r = validateQuery(mkValidQuery({ text: '' })); + expect(r.ok).toBe(false); + expect(r.issues.some(i => i.field === 'text')).toBe(true); + }); + + test('invalid tier fails', () => { + const r = validateQuery(mkValidQuery({ tier: 'invalid' as unknown as 'easy' })); + expect(r.ok).toBe(false); + expect(r.issues.some(i => i.field === 'tier')).toBe(true); + }); + + test('invalid expected_output_type fails', () => { + const r = validateQuery(mkValidQuery({ + expected_output_type: 'nonsense' as unknown as 'answer-string', + })); + expect(r.ok).toBe(false); + expect(r.issues.some(i => i.field === 'expected_output_type')).toBe(true); + }); +}); + +describe('validateQuery — temporal as_of_date rule', () => { + test('non-temporal query without as_of_date passes', () => { + const r = validateQuery(mkValidQuery({ text: 'Who founded Acme?' })); + expect(r.ok).toBe(true); + }); + + test('"is" verb without as_of_date fails', () => { + const r = validateQuery(mkValidQuery({ text: 'Where is Sarah working?' })); + expect(r.ok).toBe(false); + expect(r.issues.some(i => i.field === 'as_of_date')).toBe(true); + }); + + test('"was" verb without as_of_date fails', () => { + const r = validateQuery(mkValidQuery({ text: 'Who was at the meeting?' })); + expect(r.ok).toBe(false); + expect(r.issues.some(i => i.field === 'as_of_date')).toBe(true); + }); + + test('"as of" verb without as_of_date fails', () => { + const r = validateQuery(mkValidQuery({ text: 'As of 2024, who invested?' })); + expect(r.ok).toBe(false); + expect(r.issues.some(i => i.field === 'as_of_date')).toBe(true); + }); + + test('temporal query with "corpus-end" passes', () => { + const r = validateQuery(mkValidQuery({ + text: 'Who is the CEO now?', + as_of_date: 'corpus-end', + })); + expect(r.ok).toBe(true); + }); + + test('temporal query with "per-source" passes', () => { + const r = validateQuery(mkValidQuery({ + text: 'Who was at the meeting?', + as_of_date: 'per-source', + })); + expect(r.ok).toBe(true); + }); + + test('temporal query with ISO-8601 date passes', () => { + const r = validateQuery(mkValidQuery({ + text: 'Who was at Acme in 2023?', + as_of_date: '2023-01-01', + })); + expect(r.ok).toBe(true); + }); + + test('temporal query with bogus as_of_date fails', () => { + const r = validateQuery(mkValidQuery({ + text: 'Who is the CEO now?', + as_of_date: 'yesterday', + })); + expect(r.ok).toBe(false); + expect(r.issues.some(i => i.field === 'as_of_date')).toBe(true); + }); +}); + +describe('validateQuery — gold shape by expected_output_type', () => { + test('cited-source-pages requires gold.relevant array', () => { + const r = validateQuery(mkValidQuery({ + expected_output_type: 'cited-source-pages', + gold: {}, + })); + expect(r.ok).toBe(false); + expect(r.issues.some(i => i.field === 'gold.relevant')).toBe(true); + }); + + test('cited-source-pages with malformed slug fails', () => { + const r = validateQuery(mkValidQuery({ + gold: { relevant: ['not-a-slug'] }, + })); + expect(r.ok).toBe(false); + expect(r.issues.some(i => i.field === 'gold.relevant')).toBe(true); + }); + + test('abstention requires expected_abstention=true', () => { + const r = validateQuery(mkValidQuery({ + expected_output_type: 'abstention', + gold: {}, + })); + expect(r.ok).toBe(false); + expect(r.issues.some(i => i.field === 'gold.expected_abstention')).toBe(true); + }); + + test('abstention with expected_abstention=true passes', () => { + const r = validateQuery(mkValidQuery({ + expected_output_type: 'abstention', + gold: { expected_abstention: true }, + })); + expect(r.ok).toBe(true); + }); +}); + +describe('validateQuery — tier 5.5 author requirement', () => { + test('externally-authored without author fails', () => { + const r = validateQuery(mkValidQuery({ + tier: 'externally-authored', + })); + expect(r.ok).toBe(false); + expect(r.issues.some(i => i.field === 'author')).toBe(true); + }); + + test('externally-authored with author passes', () => { + const r = validateQuery(mkValidQuery({ + tier: 'externally-authored', + author: 'synthetic-outsider-v1', + })); + expect(r.ok).toBe(true); + }); +}); + +describe('validateQuerySet — batch level', () => { + test('duplicate ids fail', () => { + const r = validateQuerySet([ + mkValidQuery({ id: 'q-0001' }), + mkValidQuery({ id: 'q-0001' }), + ]); + expect(r.ok).toBe(false); + expect(r.issues.some(i => i.reason === 'duplicate id in batch')).toBe(true); + }); + + test('unique ids pass', () => { + const r = validateQuerySet([ + mkValidQuery({ id: 'q-0001' }), + mkValidQuery({ id: 'q-0002' }), + ]); + expect(r.ok).toBe(true); + }); + + test('issues from multiple queries aggregated', () => { + const r = validateQuerySet([ + mkValidQuery({ id: 'q-0001', text: '' }), // missing text + mkValidQuery({ id: 'q-0002', text: 'Who is CEO now?' }), // missing as_of_date + ]); + expect(r.ok).toBe(false); + expect(r.issues.some(i => i.queryId === 'q-0001' && i.field === 'text')).toBe(true); + expect(r.issues.some(i => i.queryId === 'q-0002' && i.field === 'as_of_date')).toBe(true); + }); +}); + +describe('TEMPORAL_VERBS regex', () => { + test('matches common temporal verbs', () => { + expect(TEMPORAL_VERBS.test('Where is Sarah?')).toBe(true); + expect(TEMPORAL_VERBS.test('Where was Sarah?')).toBe(true); + expect(TEMPORAL_VERBS.test('Who were the founders?')).toBe(true); + expect(TEMPORAL_VERBS.test('What was the current valuation?')).toBe(true); + expect(TEMPORAL_VERBS.test('Where is she now?')).toBe(true); + expect(TEMPORAL_VERBS.test('At the time, who led the round?')).toBe(true); + expect(TEMPORAL_VERBS.test('During Q1, which deals closed?')).toBe(true); + expect(TEMPORAL_VERBS.test('As of 2024, who works at Acme?')).toBe(true); + expect(TEMPORAL_VERBS.test('When did Alice join?')).toBe(true); + }); + + test('does not match non-temporal verbs', () => { + expect(TEMPORAL_VERBS.test('Who founded Acme?')).toBe(false); + expect(TEMPORAL_VERBS.test('Which investors backed Beta?')).toBe(false); + expect(TEMPORAL_VERBS.test('List the advisors at Gamma.')).toBe(false); + }); +}); diff --git a/eval/runner/queries/validator.ts b/eval/runner/queries/validator.ts new file mode 100644 index 000000000..653c7d9b8 --- /dev/null +++ b/eval/runner/queries/validator.ts @@ -0,0 +1,204 @@ +/** + * Runtime Query schema validator. + * + * Per the v1.1 eng pass 2 spec. Hand-rolled (no zod dep) to match existing + * gbrain codebase style (see src/core/yaml-lite.ts for precedent). + * + * Validates: + * - Required fields (id, tier, text, expected_output_type, gold) + * - Tier enum + * - expected_output_type enum + * - Temporal `as_of_date` rule: any query with a temporal verb MUST + * set as_of_date to ISO-8601 | "corpus-end" | "per-source" + * - id uniqueness within a batch + * - gold.relevant structure when the expected_output_type is + * 'cited-source-pages' (most common tier-1/2/3 pattern) + * + * Public functions: + * validateQuery(q) -> ValidationResult single-query + * validateQuerySet(qs) -> ValidationResult + * + * On failure, returns human-readable reasons with the offending query id + * so `eval:query:validate` can point contributors at the exact problem. + */ + +import type { Query, Tier, ExpectedOutputType } from '../types.ts'; + +// ─── Enums ───────────────────────────────────────────────────────── + +const VALID_TIERS: readonly Tier[] = [ + 'easy', 'medium', 'hard', 'adversarial', 'fuzzy', 'externally-authored', +] as const; + +const VALID_OUTPUT_TYPES: readonly ExpectedOutputType[] = [ + 'answer-string', + 'canonical-entity-id', + 'cited-source-pages', + 'time-qualified-answer', + 'abstention', + 'contradiction-explanation', + 'poison-flag', + 'confidence-score', +] as const; + +// ─── Temporal rule (per eng pass 2) ──────────────────────────────── + +/** + * Regex for detecting temporal verbs in query text. If any of these + * appear, the query is temporal and MUST carry an `as_of_date` field. + * Without that, scoring is ambiguous (which version of the fact is + * considered correct?). + */ +export const TEMPORAL_VERBS = + /\b(is|was|were|current|now|at the time|during|as of|when did)\b/i; + +const ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}(T.*)?$/; + +// ─── Types ───────────────────────────────────────────────────────── + +export interface ValidationIssue { + queryId: string; + field: string; + reason: string; +} + +export interface ValidationResult { + ok: boolean; + issues: ValidationIssue[]; + /** Count of queries processed (for batch). 1 for single-query validation. */ + total: number; +} + +// ─── Individual query validation ─────────────────────────────────── + +export function validateQuery(q: Query): ValidationResult { + const issues: ValidationIssue[] = []; + const qid = q.id || '(missing id)'; + + if (!q.id || typeof q.id !== 'string' || q.id.trim().length === 0) { + issues.push({ queryId: qid, field: 'id', reason: 'id must be a non-empty string (e.g. "q-0001")' }); + } + if (!q.text || typeof q.text !== 'string' || q.text.trim().length === 0) { + issues.push({ queryId: qid, field: 'text', reason: 'text must be a non-empty string' }); + } + if (!VALID_TIERS.includes(q.tier)) { + issues.push({ queryId: qid, field: 'tier', reason: `tier must be one of ${VALID_TIERS.join(', ')}` }); + } + if (!VALID_OUTPUT_TYPES.includes(q.expected_output_type)) { + issues.push({ + queryId: qid, + field: 'expected_output_type', + reason: `expected_output_type must be one of ${VALID_OUTPUT_TYPES.join(', ')}`, + }); + } + if (!q.gold || typeof q.gold !== 'object') { + issues.push({ queryId: qid, field: 'gold', reason: 'gold must be an object' }); + } + + // Temporal as-of-date rule (eng pass 2). + if (q.text && TEMPORAL_VERBS.test(q.text)) { + if (q.as_of_date === undefined || q.as_of_date === null || q.as_of_date === '') { + issues.push({ + queryId: qid, + field: 'as_of_date', + reason: + 'temporal verb detected; as_of_date required. Set to "corpus-end", "per-source", or an ISO-8601 date.', + }); + } else if ( + q.as_of_date !== 'corpus-end' && + q.as_of_date !== 'per-source' && + !ISO_DATE_RE.test(q.as_of_date) + ) { + issues.push({ + queryId: qid, + field: 'as_of_date', + reason: 'as_of_date must be "corpus-end", "per-source", or ISO-8601 (YYYY-MM-DD)', + }); + } + } + + // If expected_output_type is cited-source-pages, gold.relevant should exist + // and be a non-empty array of slug-like strings. + if (q.expected_output_type === 'cited-source-pages') { + const rel = (q.gold as Record)?.relevant; + if (!Array.isArray(rel) || rel.length === 0) { + issues.push({ + queryId: qid, + field: 'gold.relevant', + reason: 'cited-source-pages queries require gold.relevant[] with at least one slug', + }); + } else { + for (const s of rel) { + if (typeof s !== 'string' || !/^[a-z][a-z0-9-]*\/[a-z0-9][a-z0-9-]*$/.test(s)) { + issues.push({ + queryId: qid, + field: 'gold.relevant', + reason: `slug "${s}" does not match "dir/slug" format (e.g. "people/alice-chen")`, + }); + break; // one message per query is enough + } + } + } + } + + // Abstention queries MUST set expected_abstention to true. + if (q.expected_output_type === 'abstention') { + const expAb = (q.gold as Record)?.expected_abstention; + if (expAb !== true) { + issues.push({ + queryId: qid, + field: 'gold.expected_abstention', + reason: 'abstention queries require gold.expected_abstention === true', + }); + } + } + + // Tier 5.5 externally-authored queries must carry an author field. + if (q.tier === 'externally-authored') { + if (!q.author || typeof q.author !== 'string' || q.author.trim().length === 0) { + issues.push({ + queryId: qid, + field: 'author', + reason: 'externally-authored queries require an author field (e.g. "@alice-researcher" or "synthetic-outsider-v1")', + }); + } + } + + return { ok: issues.length === 0, issues, total: 1 }; +} + +// ─── Batch validation ─────────────────────────────────────────────── + +export function validateQuerySet(queries: Query[]): ValidationResult { + const issues: ValidationIssue[] = []; + const seenIds = new Set(); + + for (const q of queries) { + const r = validateQuery(q); + issues.push(...r.issues); + + // Duplicate ID check (batch-level). + if (q.id) { + if (seenIds.has(q.id)) { + issues.push({ queryId: q.id, field: 'id', reason: 'duplicate id in batch' }); + } + seenIds.add(q.id); + } + } + + return { ok: issues.length === 0, issues, total: queries.length }; +} + +// ─── Formatting helpers (for CLI output) ─────────────────────────── + +export function formatIssues(result: ValidationResult): string { + if (result.ok) { + return `\u2713 All ${result.total} queries valid.`; + } + const lines: string[] = []; + lines.push(`\u2717 ${result.issues.length} issue(s) across ${result.total} query/queries:`); + for (const issue of result.issues) { + lines.push(` [${issue.queryId}] ${issue.field}: ${issue.reason}`); + } + return lines.join('\n'); +}