feat(eval): Phase 2 query validator + Tier 5 Fuzzy + Tier 5.5 synthetic + N=5 tolerance bands

Closes multiple Phase 2 items in one commit since they form a cohesive
package: query schema enforcement + new query tiers + per-query-set
statistical rigor.

Added:
  eval/runner/queries/validator.ts               — hand-rolled Query schema validator
  eval/runner/queries/validator.test.ts          — 24 unit tests, all pass
  eval/runner/queries/tier5-fuzzy.ts             — 30 hand-authored Tier 5 Fuzzy/Vibe queries
  eval/runner/queries/tier5_5-synthetic.ts       — 50 SYNTHETIC-labeled outsider-style queries (author: "synthetic-outsider-v1")
  eval/runner/queries/index.ts                   — aggregator + validateAll()

Modified:
  eval/runner/multi-adapter.ts                   — N=5 runs per adapter (BRAINBENCH_N override), page-order shuffle, mean±stddev reporting

Query validator (hand-rolled, no zod dep to match gbrain codebase style):
  - Temporal verb regex enforces as_of_date (per eng pass 2 spec):
    /\\b(is|was|were|current|now|at the time|during|as of|when did)\\b/i
  - Validates tier enum, expected_output_type enum, gold shape per type
  - gold.relevant must be non-empty slug[] for cited-source-pages queries
  - abstention requires gold.expected_abstention === true
  - externally-authored tier requires author field
  - batch validation catches duplicate IDs

Tier 5 Fuzzy/Vibe (30 queries, hand-authored):
  - Vague recall: "Someone who was a senior engineer at a biotech company..."
  - Trait-based: "The engineer who pushed back on microservices"
  - Cultural/epithet: "Who is known as a 'systems builder' in security?"
  - Abstention bait: "Which Layer 1 project did the crypto guy leave?" (prose
    mentions but never names; good systems abstain)
  - Addresses Codex's circularity critique — vague queries where graph-heavy
    systems shouldn't inherently win.

Tier 5.5 Synthetic Outsider (50 queries, AI-authored placeholder):
  - Clearly labeled author: "synthetic-outsider-v1"
  - Phrasing variety not in the 4 template families:
    * fragment style ("crypto founder Goldman Sachs background")
    * polite/natural ("Can you pull up what we have on...")
    * comparison ("What is the difference between X and Y?")
    * follow-up ("And who else advises Orbit Labs?")
    * typos/misspellings ("adam lopez bioinformatcis")
    * similarity ("Find me someone like Alice Davis...")
    * imperative ("Pull up Alice Davis")
  - Real Tier 5.5 from outside researchers supersedes synthetic via
    PRs to eval/external-authors/ (docs ship in follow-up commit).

N=5 tolerance bands:
  - Default N=5, override via BRAINBENCH_N env var (e.g. BRAINBENCH_N=1 for dev loops)
  - Per-run seeded Fisher-Yates shuffle of page ingest order (LCG seed = run_idx+1)
  - Surfaces order-dependent adapter bugs (tie-break-by-first-seen etc.)
  - Reports mean ± sample-stddev per metric
  - "stddev = 0" is honest signal that the adapter is deterministic, not a bug.
    LLM-judge metrics (future) will naturally produce non-zero stddev.

Validation: all 80 Tier 5 + 5.5 queries pass validateAll(). 24 validator
unit tests pass.

Next commit: world.html contributor explorer (Phase 3).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

eval/runner/multi-adapter.ts

@@ -263,22 +263,69 @@ function parseRelationalQuery(
   return { seed: '', direction: 'in', linkTypes: [] };
 }
 
-// ─── Scorecard ──────────────────────────────────────────────────────
+// ─── Tolerance bands (N-run variance measurement) ──────────────────
 
-interface AdapterScorecard {
-  adapter: string;
-  queries: number;
+/**
+ * N=5 per eng pass 3 decision. For current adapters (all deterministic
+ * over sorted page input) bands will be ~0. Per-run variance surfaces
+ * when any of these enter the benchmark:
+ *   - LLM-judge scoring (future)
+ *   - Non-deterministic embedding providers
+ *   - Page-ordering-dependent dedup tie-breaks (induced here by shuffle)
+ *
+ * Shuffling ingestion order per run reveals order-sensitive bugs. An
+ * adapter with hidden order-dependence (e.g. a tie-break that favors
+ * first-seen slug) shows up as non-zero stddev.
+ */
+const RUNS_PER_ADAPTER = Number(process.env.BRAINBENCH_N ?? '5');
+
+interface RunResult {
   mean_precision_at_k: number;
   mean_recall_at_k: number;
   correct_in_top_k: number;
   total_expected: number;
 }
 
-async function scoreAdapter(
+interface AdapterScorecard {
+  adapter: string;
+  queries: number;
+  runs: number;
+  /** Mean across N runs. */
+  mean_precision_at_k: number;
+  mean_recall_at_k: number;
+  /** Sample stddev across N runs (n-1 denominator). Zero means deterministic. */
+  stddev_precision_at_k: number;
+  stddev_recall_at_k: number;
+  /** From the first run (for the headline "correct/gold" column). */
+  correct_in_top_k: number;
+  total_expected: number;
+}
+
+/**
+ * Seeded Fisher-Yates shuffle. Deterministic given the same seed so
+ * N-run results are reproducible by anyone re-running with the same seed.
+ * Uses a linear congruential generator (LCG) — good enough for benchmark
+ * permutations, not cryptographic.
+ */
+function shuffleSeeded<T>(arr: T[], seed: number): T[] {
+  const out = [...arr];
+  let s = seed >>> 0;
+  const next = () => {
+    s = (s * 1664525 + 1013904223) >>> 0;
+    return s / 0x100000000;
+  };
+  for (let i = out.length - 1; i > 0; i--) {
+    const j = Math.floor(next() * (i + 1));
+    [out[i], out[j]] = [out[j], out[i]];
+  }
+  return out;
+}
+
+async function scoreOneRun(
   adapter: Adapter,
   pages: Page[],
   queries: Query[],
-): Promise<AdapterScorecard> {
+): Promise<RunResult> {
   const state = await adapter.init(pages, { name: adapter.name });
   let totalP = 0;
   let totalR = 0;
@@ -289,14 +336,11 @@ async function scoreAdapter(
     const relevant = new Set(q.gold.relevant ?? []);
     totalP += precisionAtK(results, relevant, TOP_K);
     totalR += recallAtK(results, relevant, TOP_K);
-    // Exact top-K correct count for the headline table.
     const topK = results.slice(0, TOP_K);
     for (const r of topK) if (relevant.has(r.page_id)) totalCorrect++;
     totalExpected += relevant.size;
   }
   return {
-    adapter: adapter.name,
-    queries: queries.length,
     mean_precision_at_k: queries.length > 0 ? totalP / queries.length : 0,
     mean_recall_at_k: queries.length > 0 ? totalR / queries.length : 0,
     correct_in_top_k: totalCorrect,
@@ -304,10 +348,53 @@ async function scoreAdapter(
   };
 }
 
+function stddev(values: number[]): number {
+  const n = values.length;
+  if (n < 2) return 0;
+  const mean = values.reduce((a, b) => a + b, 0) / n;
+  const variance = values.reduce((a, b) => a + (b - mean) ** 2, 0) / (n - 1);
+  return Math.sqrt(variance);
+}
+
+async function scoreAdapter(
+  adapter: Adapter,
+  pages: Page[],
+  queries: Query[],
+): Promise<AdapterScorecard> {
+  const runResults: RunResult[] = [];
+  for (let i = 0; i < RUNS_PER_ADAPTER; i++) {
+    // Shuffle pages per run with a per-run seed. Seed = i + 1 (not 0,
+    // since LCG iterates once at start of next()). Run 0 uses the seed
+    // that produces a minimally-scrambled permutation; doesn't matter
+    // for correctness since we aggregate across runs.
+    const shuffled = shuffleSeeded(pages, i + 1);
+    const r = await scoreOneRun(adapter, shuffled, queries);
+    runResults.push(r);
+  }
+  const pVals = runResults.map(r => r.mean_precision_at_k);
+  const rVals = runResults.map(r => r.mean_recall_at_k);
+  return {
+    adapter: adapter.name,
+    queries: queries.length,
+    runs: RUNS_PER_ADAPTER,
+    mean_precision_at_k: pVals.reduce((a, b) => a + b, 0) / pVals.length,
+    mean_recall_at_k: rVals.reduce((a, b) => a + b, 0) / rVals.length,
+    stddev_precision_at_k: stddev(pVals),
+    stddev_recall_at_k: stddev(rVals),
+    correct_in_top_k: runResults[0].correct_in_top_k,
+    total_expected: runResults[0].total_expected,
+  };
+}
+
 function pct(n: number, digits = 1): string {
   return `${(n * 100).toFixed(digits)}%`;
 }
 
+function pctBand(mean: number, sd: number, digits = 1): string {
+  if (sd === 0) return pct(mean, digits);
+  return `${pct(mean, digits)} \u00b1${(sd * 100).toFixed(digits)}`;
+}
+
 // ─── Main ──────────────────────────────────────────────────────────
 
 async function main() {
@@ -336,24 +423,25 @@ async function main() {
     process.exit(1);
   }
 
-  log('## Running adapters\n');
+  log(`## Running adapters (N=${RUNS_PER_ADAPTER} runs per adapter, page-order shuffled per run)\n`);
   const scorecards: AdapterScorecard[] = [];
   for (const a of adapters) {
     log(`- ${a.name} ...`);
     const t0 = Date.now();
     const sc = await scoreAdapter(a, pages, queries);
     const elapsed = ((Date.now() - t0) / 1000).toFixed(1);
-    log(`  done (${elapsed}s). P@${TOP_K} ${pct(sc.mean_precision_at_k)}, R@${TOP_K} ${pct(sc.mean_recall_at_k)}, ${sc.correct_in_top_k}/${sc.total_expected} correct in top-${TOP_K}`);
+    log(`  done (${elapsed}s). P@${TOP_K} ${pctBand(sc.mean_precision_at_k, sc.stddev_precision_at_k)}, R@${TOP_K} ${pctBand(sc.mean_recall_at_k, sc.stddev_recall_at_k)}, ${sc.correct_in_top_k}/${sc.total_expected} correct (run 1)`);
     scorecards.push(sc);
   }
 
-  log('\n## Side-by-side scorecard\n');
-  log(`| Adapter             | Queries | P@${TOP_K}  | R@${TOP_K}  | Correct in top-${TOP_K} | Gold total |`);
-  log('|---------------------|---------|--------|--------|---------------------|------------|');
+  log('\n## Side-by-side scorecard (mean \u00b1 stddev across N runs)\n');
+  log(`| Adapter             | Runs | Queries | P@${TOP_K} (mean \u00b1 sd)    | R@${TOP_K} (mean \u00b1 sd)    |`);
+  log('|---------------------|------|---------|---------------------|---------------------|');
   for (const sc of scorecards) {
-    log(`| ${sc.adapter.padEnd(19)} | ${String(sc.queries).padStart(7)} | ${pct(sc.mean_precision_at_k).padStart(6)} | ${pct(sc.mean_recall_at_k).padStart(6)} | ${String(sc.correct_in_top_k).padStart(19)} | ${String(sc.total_expected).padStart(10)} |`);
+    log(`| ${sc.adapter.padEnd(19)} | ${String(sc.runs).padStart(4)} | ${String(sc.queries).padStart(7)} | ${pctBand(sc.mean_precision_at_k, sc.stddev_precision_at_k).padStart(19)} | ${pctBand(sc.mean_recall_at_k, sc.stddev_recall_at_k).padStart(19)} |`);
   }
   log('');
+  log('*Stddev = 0 means the adapter is deterministic over page ordering. Non-zero stddev surfaces order-dependent bugs (e.g. tie-break that favors first-seen slug). LLM-judge-based metrics will produce non-zero stddev once added.*\n');
 
   if (scorecards.length >= 2) {
     const [first, ...rest] = scorecards;

eval/runner/queries/index.ts

@@ -0,0 +1,38 @@
+/**
+ * Aggregates all tier-5/5.5 query sets and exposes validator helpers.
+ *
+ * Usage:
+ *   import { getAllTierQueries, validateAll } from './queries';
+ *   const queries = getAllTierQueries();
+ *   const result = validateAll(queries);
+ */
+
+import type { Query } from '../types.ts';
+import { getTier5FuzzyQueries } from './tier5-fuzzy.ts';
+import { getTier5_5SyntheticQueries } from './tier5_5-synthetic.ts';
+import { validateQuerySet, formatIssues } from './validator.ts';
+
+/** Tier 5 Fuzzy/Vibe (hand-authored by gstack maintainers). */
+export { getTier5FuzzyQueries } from './tier5-fuzzy.ts';
+
+/** Tier 5.5 externally-authored (SYNTHETIC placeholder; see CONTRIBUTING.md). */
+export { getTier5_5SyntheticQueries } from './tier5_5-synthetic.ts';
+
+export { validateQuery, validateQuerySet, formatIssues, TEMPORAL_VERBS } from './validator.ts';
+export type { ValidationIssue, ValidationResult } from './validator.ts';
+
+/** All Tier 5 + 5.5 queries concatenated. */
+export function getAllTierQueries(): Query[] {
+  return [...getTier5FuzzyQueries(), ...getTier5_5SyntheticQueries()];
+}
+
+/** Validate the complete Tier-5 + 5.5 set (used by CI + eval:query:validate). */
+export function validateAll(): { ok: boolean; count: number; report: string } {
+  const queries = getAllTierQueries();
+  const result = validateQuerySet(queries);
+  return {
+    ok: result.ok,
+    count: queries.length,
+    report: formatIssues(result),
+  };
+}

eval/runner/queries/tier5-fuzzy.ts

@@ -0,0 +1,308 @@
+/**
+ * Tier 5: Fuzzy / Vibe queries.
+ *
+ * Vague recall and "I know I mentioned this somewhere" — the kind of query
+ * real people ask their brain when they can't quite remember the exact
+ * entity name. Graph-heavy systems shouldn't have an inherent edge here
+ * because the query doesn't mention the target entity precisely.
+ *
+ * Per the 4-review arc: these address Codex's circularity critique
+ * ("gbrain's adversarial list is its product roadmap"). If gbrain loses
+ * ground on vague queries while winning on relational ones, that's an
+ * honest tradeoff story. If gbrain wins on BOTH, that's a stronger
+ * benchmark claim.
+ *
+ * Target: ~30 queries (statistical floor per eng pass 3).
+ *
+ * Gold derivation: each query specifies an expected answer slug set based
+ * on the canonical world (eval/data/world-v1/_ledger.json). We fix a small
+ * set of landmarks here rather than deriving from _facts — fuzzy queries
+ * don't map 1:1 to _facts fields.
+ */
+
+import type { Query } from '../types.ts';
+
+/**
+ * Hand-authored Tier 5 vibe queries. Each targets real entities from
+ * eval/data/world-v1/ that actually exist in the corpus. Slugs verified
+ * against world-v1 shards at authoring time.
+ */
+export const TIER5_FUZZY_QUERIES: Query[] = [
+  // ── "I know I mentioned this somewhere" style ─────────────────────
+  {
+    id: 'q5-0001',
+    tier: 'fuzzy',
+    text: 'Someone I know was a senior engineer at a biotech company doing drug discovery — who?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lopez-113'] },
+    as_of_date: 'per-source',
+    tags: ['vague-recall', 'role-based'],
+    known_failure_modes: ['might return all biotech company pages; we want the person'],
+  },
+  {
+    id: 'q5-0002',
+    tier: 'fuzzy',
+    text: 'The crypto-infra founder who did a stint at Goldman before building his own thing',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lee-19'] },
+    tags: ['vague-recall', 'biographical'],
+  },
+  {
+    id: 'q5-0003',
+    tier: 'fuzzy',
+    text: 'The security advisor woman based in Boston, multi-year engagement with a cybersecurity startup',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/alice-davis-172'] },
+    tags: ['vague-recall', 'role-based', 'location'],
+  },
+
+  // ── Summarization-over-messy-notes style ──────────────────────────
+  {
+    id: 'q5-0004',
+    tier: 'fuzzy',
+    text: 'Summarize what we know about founders who raised Series A in 2024.',
+    expected_output_type: 'answer-string',
+    gold: { relevant: [] },
+    acceptable_variants: ['Series A 2024 founders summary'],
+    tags: ['summarization', 'multi-entity'],
+    known_failure_modes: ['accept any top-K that includes actual Series-A-2024 founders'],
+  },
+  {
+    id: 'q5-0005',
+    tier: 'fuzzy',
+    text: 'Who are the fintech advisors in our network?',
+    expected_output_type: 'abstention',
+    gold: { expected_abstention: true },
+    tags: ['summarization', 'role-intersection', 'partial-in-corpus'],
+    known_failure_modes: ['"fintech" advisors are scarce in twin-amara corpus; a good system abstains or flags partial match'],
+  },
+  {
+    id: 'q5-0006',
+    tier: 'fuzzy',
+    text: 'Tell me about the people who push back hard on microservices',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lopez-113'] },
+    tags: ['trait-based', 'opinion'],
+    known_failure_modes: ['requires catching "controversial internal memo on microservices" in prose'],
+  },
+
+  // ── Partial-information recall ────────────────────────────────────
+  {
+    id: 'q5-0007',
+    tier: 'fuzzy',
+    text: 'Who has a "40 under 40" mention?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lee-19'] },
+    tags: ['biographical-fragment'],
+  },
+  {
+    id: 'q5-0008',
+    tier: 'fuzzy',
+    text: 'The company whose CEO insists on founder-friendly terms and minimal board seats',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['companies/forge-19'] },
+    tags: ['trait-based', 'culture'],
+  },
+  {
+    id: 'q5-0009',
+    tier: 'fuzzy',
+    text: 'Someone we know who cut sequencing pipeline runtime by about 40%',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lopez-113'] },
+    tags: ['achievement-based'],
+  },
+  {
+    id: 'q5-0010',
+    tier: 'fuzzy',
+    text: 'The person who wrote an internal memo about deleting half the microservices',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lopez-113'] },
+    tags: ['behavioral-recall'],
+  },
+
+  // ── "what was that thing about..." style ──────────────────────────
+  {
+    id: 'q5-0011',
+    tier: 'fuzzy',
+    text: 'What was the thing about MEV-resistant transaction ordering?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lee-19'] },
+    as_of_date: 'per-source',
+    tags: ['topic-recall'],
+  },
+  {
+    id: 'q5-0012',
+    tier: 'fuzzy',
+    text: 'The partner who focuses on early-stage fintech',
+    expected_output_type: 'abstention',
+    gold: { expected_abstention: true },
+    tags: ['role-based', 'domain', 'partial-in-corpus'],
+    known_failure_modes: ['multiple partial matches; good systems either abstain or flag "multiple candidates"'],
+  },
+  {
+    id: 'q5-0013',
+    tier: 'fuzzy',
+    text: 'Which Layer 1 project did that crypto guy leave over tokenomics disagreements?',
+    expected_output_type: 'abstention',
+    gold: { expected_abstention: true },
+    known_failure_modes: ['prose says a Layer 1 project but never names it; good systems abstain'],
+    tags: ['abstention', 'under-specified'],
+  },
+  {
+    id: 'q5-0014',
+    tier: 'fuzzy',
+    text: 'Who built that cross-chain messaging protocol?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['companies/forge-19'] },
+    tags: ['product-feature-recall'],
+  },
+  {
+    id: 'q5-0015',
+    tier: 'fuzzy',
+    text: 'The engineer who is notoriously demanding on code review',
+    expected_output_type: 'abstention',
+    gold: { expected_abstention: true },
+    as_of_date: 'corpus-end',
+    known_failure_modes: ['no single canonical answer in corpus; good systems flag ambiguity'],
+    tags: ['trait-based', 'partial-in-corpus'],
+  },
+
+  // ── Emotional / cultural recall ──────────────────────────────────
+  {
+    id: 'q5-0016',
+    tier: 'fuzzy',
+    text: 'The company whose culture is described as "either loved or hated"',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['companies/forge-19'] },
+    as_of_date: 'per-source',
+    tags: ['culture-recall'],
+  },
+  {
+    id: 'q5-0017',
+    tier: 'fuzzy',
+    text: 'The advisor who pushed hard for zero-trust architecture overhaul',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/alice-davis-172'] },
+    tags: ['behavior-recall', 'advocacy'],
+  },
+  {
+    id: 'q5-0018',
+    tier: 'fuzzy',
+    text: 'Someone who speaks selectively at conferences and prefers small venues',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/alice-davis-172'] },
+    tags: ['preference-recall'],
+  },
+  {
+    id: 'q5-0019',
+    tier: 'fuzzy',
+    text: 'Who is rumored to be writing a book on security culture?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/alice-davis-172'] },
+    as_of_date: 'corpus-end',
+    tags: ['gossip', 'side-project'],
+  },
+
+  // ── "something about X" generic-topic style ──────────────────────
+  {
+    id: 'q5-0020',
+    tier: 'fuzzy',
+    text: 'Any portfolio companies focused on drug discovery?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['companies/delta-3'] },
+    tags: ['topical', 'industry'],
+  },
+  {
+    id: 'q5-0021',
+    tier: 'fuzzy',
+    text: 'Who among our founders worked at Goldman Sachs?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lee-19'] },
+    tags: ['background-intersection'],
+  },
+  {
+    id: 'q5-0022',
+    tier: 'fuzzy',
+    text: 'The person with an MIT CS background who dropped out of a PhD',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lee-19'] },
+    tags: ['biography-fragment'],
+  },
+  {
+    id: 'q5-0023',
+    tier: 'fuzzy',
+    text: 'Who among our people is a long-distance runner?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lopez-113'] },
+    as_of_date: 'corpus-end',
+    tags: ['personal-trait'],
+  },
+
+  // ── Negative / abstention fuzzy (known failure bait) ─────────────
+  {
+    id: 'q5-0024',
+    tier: 'fuzzy',
+    text: 'Which YC W18 founder built an analytics dashboard?',
+    expected_output_type: 'abstention',
+    gold: { expected_abstention: true },
+    tags: ['abstention', 'not-in-corpus'],
+    known_failure_modes: ['W18 batch doesn\'t exist in this corpus; good systems abstain'],
+  },
+  {
+    id: 'q5-0025',
+    tier: 'fuzzy',
+    text: 'Who founded the developer-tools company that got acquihired by Roche?',
+    expected_output_type: 'abstention',
+    gold: { expected_abstention: true },
+    tags: ['abstention', 'mentioned-but-not-named'],
+    known_failure_modes: ['prose mentions a bioinformatics startup acquired by Roche but never names it; good systems abstain'],
+  },
+
+  // ── Cross-referencing without exact entity names ─────────────────
+  {
+    id: 'q5-0026',
+    tier: 'fuzzy',
+    text: 'What companies have enterprise security architecture help from an outside advisor?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['companies/prism-43'] },
+    tags: ['relational-fuzzy'],
+  },
+  {
+    id: 'q5-0027',
+    tier: 'fuzzy',
+    text: 'Who is known as a "systems builder" in the security space?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/alice-davis-172'] },
+    as_of_date: 'corpus-end',
+    tags: ['epithet-recall'],
+  },
+  {
+    id: 'q5-0028',
+    tier: 'fuzzy',
+    text: 'The Boston-based person who completed a SOC 2 audit for someone',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/alice-davis-172'] },
+    tags: ['achievement-plus-location'],
+  },
+  {
+    id: 'q5-0029',
+    tier: 'fuzzy',
+    text: 'Which advisor has SecureCon Northeast speaking experience?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/alice-davis-172'] },
+    tags: ['conference-history'],
+  },
+  {
+    id: 'q5-0030',
+    tier: 'fuzzy',
+    text: 'Someone who published multiple technical papers on cryptographic primitives',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lee-19'] },
+    tags: ['academic-output'],
+  },
+];
+
+export function getTier5FuzzyQueries(): Query[] {
+  // Defensive copy so callers can't mutate the canonical set.
+  return TIER5_FUZZY_QUERIES.map(q => ({ ...q, gold: { ...q.gold }, tags: q.tags ? [...q.tags] : undefined }));
+}

eval/runner/queries/tier5_5-synthetic.ts

@@ -0,0 +1,346 @@
+/**
+ * Tier 5.5: Externally-Authored queries (SYNTHETIC placeholder).
+ *
+ * Per plan file: real Tier 5.5 requires 2-3 outside researchers writing
+ * ~50 queries each against the committed corpus. That's a human-in-the-loop
+ * deliverable (see eval/CONTRIBUTING.md for how to submit).
+ *
+ * This file ships a SYNTHETIC placeholder set: AI-authored queries that
+ * deliberately vary phrasing patterns to simulate what an outsider would
+ * write. They are CLEARLY labeled via `author: "synthetic-outsider-v1"`
+ * so real researcher submissions can supersede them without ambiguity.
+ *
+ * Why ship synthetic ones at all:
+ *   - Tier 5.5 has to exist in the scorecard for the multi-axis report
+ *     to have a full column. A missing tier reads as "not measured."
+ *   - The synthetic set establishes phrasing variety (full sentences,
+ *     short fragments, follow-up style, comparison style, "what's the
+ *     difference between X and Y" style) that the 4 template families
+ *     in the medium tier don't cover.
+ *   - When real researchers submit, scorecards can compare "AI-authored"
+ *     vs "human-authored" columns to flag where LLM judgment differs
+ *     from human judgment on the same corpus.
+ *
+ * Author field: "synthetic-outsider-v1" for every query here.
+ *
+ * Gold verification: slugs referenced below exist in eval/data/world-v1/.
+ * Small slug set for authenticity; entity pages cited: adam-lee-19,
+ * adam-lopez-113, alice-davis-172, forge-19, delta-3, prism-43,
+ * orbit-labs-92 (+ others from the corpus).
+ */
+
+import type { Query } from '../types.ts';
+
+const AUTHOR = 'synthetic-outsider-v1';
+
+export const TIER5_5_SYNTHETIC_QUERIES: Query[] = [
+  // ─── Short-fragment style (how real researchers write notes) ─────
+  { id: 'q55-0001', tier: 'externally-authored', author: AUTHOR,
+    text: 'crypto founder Goldman Sachs background',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lee-19'] },
+    tags: ['fragment-style'] },
+  { id: 'q55-0002', tier: 'externally-authored', author: AUTHOR,
+    text: 'Prism cybersecurity advisor engagement history',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/alice-davis-172', 'companies/prism-43'] },
+    tags: ['fragment-style', 'relational'] },
+  { id: 'q55-0003', tier: 'externally-authored', author: AUTHOR,
+    text: 'Delta biotech engineers',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lopez-113'] },
+    tags: ['fragment-style'] },
+  { id: 'q55-0004', tier: 'externally-authored', author: AUTHOR,
+    text: 'Forge crypto infrastructure founder details',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lee-19', 'companies/forge-19'] },
+    tags: ['fragment-style', 'founder-company'] },
+
+  // ─── Full-sentence style with natural hedging ─────────────────────
+  { id: 'q55-0005', tier: 'externally-authored', author: AUTHOR,
+    text: 'Can you pull up what we have on the founder who left a Layer 1 project over tokenomics?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lee-19'] },
+    tags: ['polite-natural'] },
+  { id: 'q55-0006', tier: 'externally-authored', author: AUTHOR,
+    text: 'I need the background on the Delta senior engineer who cut pipeline runtime',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lopez-113'] },
+    tags: ['polite-natural', 'specific-achievement'] },
+  { id: 'q55-0007', tier: 'externally-authored', author: AUTHOR,
+    text: 'Please find the advisor with SOC 2 audit prep experience at Orbit Labs',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/alice-davis-172'] },
+    tags: ['polite-natural', 'relational'] },
+
+  // ─── Comparison / differentiation style ───────────────────────────
+  { id: 'q55-0008', tier: 'externally-authored', author: AUTHOR,
+    text: 'What is the difference between Adam Lee and Adam Lopez in our network?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lee-19', 'people/adam-lopez-113'] },
+    as_of_date: 'corpus-end',
+    tags: ['disambiguation', 'comparison'] },
+  { id: 'q55-0009', tier: 'externally-authored', author: AUTHOR,
+    text: 'Compare Forge and Delta as companies in our portfolio',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['companies/forge-19', 'companies/delta-3'] },
+    tags: ['comparison'] },
+  { id: 'q55-0010', tier: 'externally-authored', author: AUTHOR,
+    text: 'Which of our advisors is based on the East Coast?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/alice-davis-172'] },
+    as_of_date: 'corpus-end',
+    tags: ['location-filter'] },
+
+  // ─── Who-does-what style (role-first) ──────────────────────────────
+  { id: 'q55-0011', tier: 'externally-authored', author: AUTHOR,
+    text: 'Who focuses on synthetic biology at Delta?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lopez-113'] },
+    tags: ['role-first'] },
+  { id: 'q55-0012', tier: 'externally-authored', author: AUTHOR,
+    text: 'Who wrote the whitepaper on MEV-resistant transaction ordering?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lee-19'] },
+    tags: ['achievement-first'] },
+  { id: 'q55-0013', tier: 'externally-authored', author: AUTHOR,
+    text: 'Who would you ask about enterprise security architecture?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/alice-davis-172'] },
+    tags: ['skill-lookup'] },
+  { id: 'q55-0014', tier: 'externally-authored', author: AUTHOR,
+    text: 'Who is the expert on bioinformatics pipelines in our network?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lopez-113'] },
+    as_of_date: 'corpus-end',
+    tags: ['expertise-lookup'] },
+
+  // ─── Follow-up style (assumes prior context) ──────────────────────
+  { id: 'q55-0015', tier: 'externally-authored', author: AUTHOR,
+    text: 'And who else advises Orbit Labs?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/alice-davis-172'] },
+    tags: ['follow-up', 'assumed-context'] },
+  { id: 'q55-0016', tier: 'externally-authored', author: AUTHOR,
+    text: 'Also at Delta?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lopez-113'] },
+    tags: ['follow-up', 'minimal'] },
+
+  // ─── Characteristic / trait recall ─────────────────────────────────
+  { id: 'q55-0017', tier: 'externally-authored', author: AUTHOR,
+    text: 'Our demanding engineering leader with the long-term vision approach',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lee-19'] },
+    tags: ['trait-description'] },
+  { id: 'q55-0018', tier: 'externally-authored', author: AUTHOR,
+    text: 'The fast-shipping opinionated engineer (likes Postgres, hates meetings)',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lopez-113'] },
+    tags: ['multi-trait'] },
+  { id: 'q55-0019', tier: 'externally-authored', author: AUTHOR,
+    text: 'A systems-builder style advisor who scales security architecture',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/alice-davis-172'] },
+    tags: ['epithet-plus-role'] },
+
+  // ─── Misspellings / typos (real researchers make these) ───────────
+  { id: 'q55-0020', tier: 'externally-authored', author: AUTHOR,
+    text: 'adam lee the crypto guy',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lee-19'] },
+    tags: ['lowercase', 'minimal-name'] },
+  { id: 'q55-0021', tier: 'externally-authored', author: AUTHOR,
+    text: 'alice davis cybersecuirty advisor',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/alice-davis-172'] },
+    tags: ['typo', 'lowercase'] },
+  { id: 'q55-0022', tier: 'externally-authored', author: AUTHOR,
+    text: 'adam lopez bioinformatcis',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lopez-113'] },
+    tags: ['typo'] },
+
+  // ─── "Find me someone like..." semantic-similarity style ─────────
+  { id: 'q55-0023', tier: 'externally-authored', author: AUTHOR,
+    text: 'Find me someone like Alice Davis for our new security engagement',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/alice-davis-172'] },
+    tags: ['recommendation', 'similarity'] },
+  { id: 'q55-0024', tier: 'externally-authored', author: AUTHOR,
+    text: 'Who has a profile similar to Adam Lopez — fast shipper, infrastructure background?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lopez-113'] },
+    tags: ['similarity', 'profile-match'] },
+
+  // ─── Negative / "is there anyone..." phrasing ────────────────────
+  { id: 'q55-0025', tier: 'externally-authored', author: AUTHOR,
+    text: 'Is there anyone in our network who has given a Mainnet conference keynote?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lee-19'] },
+    as_of_date: 'corpus-end',
+    tags: ['existence-check'] },
+  { id: 'q55-0026', tier: 'externally-authored', author: AUTHOR,
+    text: 'Does anyone know someone who has published on cryptographic primitives?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lee-19'] },
+    tags: ['existence-check', 'topic-lookup'] },
+
+  // ─── "What does X do?" direct-entity-name style ──────────────────
+  { id: 'q55-0027', tier: 'externally-authored', author: AUTHOR,
+    text: 'What does Forge do?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['companies/forge-19'] },
+    tags: ['direct-lookup'] },
+  { id: 'q55-0028', tier: 'externally-authored', author: AUTHOR,
+    text: 'What is Prism working on these days?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['companies/prism-43'] },
+    as_of_date: 'corpus-end',
+    tags: ['direct-lookup', 'temporal-latest'] },
+  { id: 'q55-0029', tier: 'externally-authored', author: AUTHOR,
+    text: "Tell me about Delta's drug discovery platform",
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['companies/delta-3'] },
+    tags: ['direct-lookup', 'domain-specific'] },
+
+  // ─── Cross-cutting relationship queries ─────────────────────────
+  { id: 'q55-0030', tier: 'externally-authored', author: AUTHOR,
+    text: 'People we know who are associated with both biotech and software infrastructure',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lopez-113'] },
+    tags: ['cross-domain'] },
+  { id: 'q55-0031', tier: 'externally-authored', author: AUTHOR,
+    text: 'Companies where our advisors have multi-year ongoing relationships',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['companies/prism-43'] },
+    tags: ['relationship-depth'] },
+
+  // ─── Role-plus-industry intersection ──────────────────────────────
+  { id: 'q55-0032', tier: 'externally-authored', author: AUTHOR,
+    text: 'Any security-focused advisors for enterprise clients?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/alice-davis-172'] },
+    tags: ['role-industry-intersect'] },
+  { id: 'q55-0033', tier: 'externally-authored', author: AUTHOR,
+    text: 'Senior infrastructure engineers in synthetic biology',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lopez-113'] },
+    tags: ['role-industry-intersect'] },
+
+  // ─── "Pull up..." imperative style ───────────────────────────────
+  { id: 'q55-0034', tier: 'externally-authored', author: AUTHOR,
+    text: 'Pull up Alice Davis',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/alice-davis-172'] },
+    tags: ['imperative', 'direct-name'] },
+  { id: 'q55-0035', tier: 'externally-authored', author: AUTHOR,
+    text: 'Show me the Forge page',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['companies/forge-19'] },
+    tags: ['imperative', 'direct-page'] },
+
+  // ─── Background-check style ──────────────────────────────────────
+  { id: 'q55-0036', tier: 'externally-authored', author: AUTHOR,
+    text: 'Prior experience of Delta senior engineers before joining',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lopez-113'] },
+    tags: ['background-check'] },
+  { id: 'q55-0037', tier: 'externally-authored', author: AUTHOR,
+    text: 'Educational background of Forge founder',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lee-19'] },
+    tags: ['biographical'] },
+
+  // ─── "When did X..." abstention-adjacent ─────────────────────────
+  { id: 'q55-0038', tier: 'externally-authored', author: AUTHOR,
+    text: 'When did Adam Lopez earn his masters degree?',
+    expected_output_type: 'abstention',
+    gold: { expected_abstention: true },
+    as_of_date: 'corpus-end',
+    tags: ['abstention', 'not-in-corpus'],
+    known_failure_modes: ['no mention of Lopez getting a masters; abstain'] },
+  { id: 'q55-0039', tier: 'externally-authored', author: AUTHOR,
+    text: 'Was Alice Davis ever at Palo Alto Networks?',
+    expected_output_type: 'abstention',
+    gold: { expected_abstention: true },
+    as_of_date: 'corpus-end',
+    tags: ['abstention', 'speculation-bait'] },
+
+  // ─── Aggregation queries ─────────────────────────────────────────
+  { id: 'q55-0040', tier: 'externally-authored', author: AUTHOR,
+    text: 'List all advisors in our corpus',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/alice-davis-172'] },
+    tags: ['aggregation', 'partial-gold'],
+    known_failure_modes: ['partial gold; accept any adviser-role pages in top-K'] },
+  { id: 'q55-0041', tier: 'externally-authored', author: AUTHOR,
+    text: 'All senior engineers',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lopez-113'] },
+    tags: ['aggregation', 'role-filter'] },
+
+  // ─── Topic-with-no-entity-name (true semantic) ───────────────────
+  { id: 'q55-0042', tier: 'externally-authored', author: AUTHOR,
+    text: 'Zero-trust architecture work',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/alice-davis-172', 'companies/prism-43'] },
+    tags: ['topic-only'] },
+  { id: 'q55-0043', tier: 'externally-authored', author: AUTHOR,
+    text: 'Mainnet cross-chain messaging',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['companies/forge-19'] },
+    tags: ['topic-only'] },
+  { id: 'q55-0044', tier: 'externally-authored', author: AUTHOR,
+    text: 'Protein modeling integration work',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lopez-113', 'companies/delta-3'] },
+    tags: ['topic-only'] },
+
+  // ─── Natural-language "tell me about..." long form ───────────────
+  { id: 'q55-0045', tier: 'externally-authored', author: AUTHOR,
+    text: 'Can you tell me about the Forge team structure and how they think about hiring?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['companies/forge-19', 'people/adam-lee-19'] },
+    tags: ['long-form'] },
+  { id: 'q55-0046', tier: 'externally-authored', author: AUTHOR,
+    text: "I want to understand Alice Davis's advisory approach and her involvement at Prism",
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/alice-davis-172', 'companies/prism-43'] },
+    tags: ['long-form', 'multi-entity'] },
+
+  // ─── Temporal / as-of queries (forces validator to require as_of_date) ─
+  { id: 'q55-0047', tier: 'externally-authored', author: AUTHOR,
+    text: 'Was Alice Davis renewing her advisory contract with Prism recently?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/alice-davis-172'] },
+    as_of_date: 'corpus-end',
+    tags: ['temporal'] },
+  { id: 'q55-0048', tier: 'externally-authored', author: AUTHOR,
+    text: 'When did Forge close their Series A?',
+    expected_output_type: 'time-qualified-answer',
+    gold: { expected_answer: '2024-06-19' },
+    as_of_date: 'corpus-end',
+    tags: ['temporal', 'exact-date'] },
+  { id: 'q55-0049', tier: 'externally-authored', author: AUTHOR,
+    text: 'What was Adam Lopez doing before he joined Delta?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lopez-113'] },
+    as_of_date: 'per-source',
+    tags: ['temporal', 'biographical'] },
+
+  // ─── "Give me a short summary" minimal output ────────────────────
+  { id: 'q55-0050', tier: 'externally-authored', author: AUTHOR,
+    text: 'Short summary of Adam Lee',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/adam-lee-19'] },
+    tags: ['summary-request'] },
+];
+
+export function getTier5_5SyntheticQueries(): Query[] {
+  return TIER5_5_SYNTHETIC_QUERIES.map(q => ({
+    ...q,
+    gold: { ...q.gold },
+    tags: q.tags ? [...q.tags] : undefined,
+  }));
+}

eval/runner/queries/validator.test.ts

@@ -0,0 +1,209 @@
+import { describe, test, expect } from 'bun:test';
+import { validateQuery, validateQuerySet, TEMPORAL_VERBS } from './validator.ts';
+import type { Query } from '../types.ts';
+
+function mkValidQuery(overrides: Partial<Query> = {}): Query {
+  return {
+    id: 'q-0001',
+    tier: 'easy',
+    text: 'Who founded Acme?',
+    expected_output_type: 'cited-source-pages',
+    gold: { relevant: ['people/alice-chen'] },
+    ...overrides,
+  };
+}
+
+describe('validateQuery — required fields', () => {
+  test('valid query passes', () => {
+    const r = validateQuery(mkValidQuery());
+    expect(r.ok).toBe(true);
+    expect(r.issues.length).toBe(0);
+  });
+
+  test('missing id fails', () => {
+    const r = validateQuery(mkValidQuery({ id: '' }));
+    expect(r.ok).toBe(false);
+    expect(r.issues.some(i => i.field === 'id')).toBe(true);
+  });
+
+  test('missing text fails', () => {
+    const r = validateQuery(mkValidQuery({ text: '' }));
+    expect(r.ok).toBe(false);
+    expect(r.issues.some(i => i.field === 'text')).toBe(true);
+  });
+
+  test('invalid tier fails', () => {
+    const r = validateQuery(mkValidQuery({ tier: 'invalid' as unknown as 'easy' }));
+    expect(r.ok).toBe(false);
+    expect(r.issues.some(i => i.field === 'tier')).toBe(true);
+  });
+
+  test('invalid expected_output_type fails', () => {
+    const r = validateQuery(mkValidQuery({
+      expected_output_type: 'nonsense' as unknown as 'answer-string',
+    }));
+    expect(r.ok).toBe(false);
+    expect(r.issues.some(i => i.field === 'expected_output_type')).toBe(true);
+  });
+});
+
+describe('validateQuery — temporal as_of_date rule', () => {
+  test('non-temporal query without as_of_date passes', () => {
+    const r = validateQuery(mkValidQuery({ text: 'Who founded Acme?' }));
+    expect(r.ok).toBe(true);
+  });
+
+  test('"is" verb without as_of_date fails', () => {
+    const r = validateQuery(mkValidQuery({ text: 'Where is Sarah working?' }));
+    expect(r.ok).toBe(false);
+    expect(r.issues.some(i => i.field === 'as_of_date')).toBe(true);
+  });
+
+  test('"was" verb without as_of_date fails', () => {
+    const r = validateQuery(mkValidQuery({ text: 'Who was at the meeting?' }));
+    expect(r.ok).toBe(false);
+    expect(r.issues.some(i => i.field === 'as_of_date')).toBe(true);
+  });
+
+  test('"as of" verb without as_of_date fails', () => {
+    const r = validateQuery(mkValidQuery({ text: 'As of 2024, who invested?' }));
+    expect(r.ok).toBe(false);
+    expect(r.issues.some(i => i.field === 'as_of_date')).toBe(true);
+  });
+
+  test('temporal query with "corpus-end" passes', () => {
+    const r = validateQuery(mkValidQuery({
+      text: 'Who is the CEO now?',
+      as_of_date: 'corpus-end',
+    }));
+    expect(r.ok).toBe(true);
+  });
+
+  test('temporal query with "per-source" passes', () => {
+    const r = validateQuery(mkValidQuery({
+      text: 'Who was at the meeting?',
+      as_of_date: 'per-source',
+    }));
+    expect(r.ok).toBe(true);
+  });
+
+  test('temporal query with ISO-8601 date passes', () => {
+    const r = validateQuery(mkValidQuery({
+      text: 'Who was at Acme in 2023?',
+      as_of_date: '2023-01-01',
+    }));
+    expect(r.ok).toBe(true);
+  });
+
+  test('temporal query with bogus as_of_date fails', () => {
+    const r = validateQuery(mkValidQuery({
+      text: 'Who is the CEO now?',
+      as_of_date: 'yesterday',
+    }));
+    expect(r.ok).toBe(false);
+    expect(r.issues.some(i => i.field === 'as_of_date')).toBe(true);
+  });
+});
+
+describe('validateQuery — gold shape by expected_output_type', () => {
+  test('cited-source-pages requires gold.relevant array', () => {
+    const r = validateQuery(mkValidQuery({
+      expected_output_type: 'cited-source-pages',
+      gold: {},
+    }));
+    expect(r.ok).toBe(false);
+    expect(r.issues.some(i => i.field === 'gold.relevant')).toBe(true);
+  });
+
+  test('cited-source-pages with malformed slug fails', () => {
+    const r = validateQuery(mkValidQuery({
+      gold: { relevant: ['not-a-slug'] },
+    }));
+    expect(r.ok).toBe(false);
+    expect(r.issues.some(i => i.field === 'gold.relevant')).toBe(true);
+  });
+
+  test('abstention requires expected_abstention=true', () => {
+    const r = validateQuery(mkValidQuery({
+      expected_output_type: 'abstention',
+      gold: {},
+    }));
+    expect(r.ok).toBe(false);
+    expect(r.issues.some(i => i.field === 'gold.expected_abstention')).toBe(true);
+  });
+
+  test('abstention with expected_abstention=true passes', () => {
+    const r = validateQuery(mkValidQuery({
+      expected_output_type: 'abstention',
+      gold: { expected_abstention: true },
+    }));
+    expect(r.ok).toBe(true);
+  });
+});
+
+describe('validateQuery — tier 5.5 author requirement', () => {
+  test('externally-authored without author fails', () => {
+    const r = validateQuery(mkValidQuery({
+      tier: 'externally-authored',
+    }));
+    expect(r.ok).toBe(false);
+    expect(r.issues.some(i => i.field === 'author')).toBe(true);
+  });
+
+  test('externally-authored with author passes', () => {
+    const r = validateQuery(mkValidQuery({
+      tier: 'externally-authored',
+      author: 'synthetic-outsider-v1',
+    }));
+    expect(r.ok).toBe(true);
+  });
+});
+
+describe('validateQuerySet — batch level', () => {
+  test('duplicate ids fail', () => {
+    const r = validateQuerySet([
+      mkValidQuery({ id: 'q-0001' }),
+      mkValidQuery({ id: 'q-0001' }),
+    ]);
+    expect(r.ok).toBe(false);
+    expect(r.issues.some(i => i.reason === 'duplicate id in batch')).toBe(true);
+  });
+
+  test('unique ids pass', () => {
+    const r = validateQuerySet([
+      mkValidQuery({ id: 'q-0001' }),
+      mkValidQuery({ id: 'q-0002' }),
+    ]);
+    expect(r.ok).toBe(true);
+  });
+
+  test('issues from multiple queries aggregated', () => {
+    const r = validateQuerySet([
+      mkValidQuery({ id: 'q-0001', text: '' }),        // missing text
+      mkValidQuery({ id: 'q-0002', text: 'Who is CEO now?' }), // missing as_of_date
+    ]);
+    expect(r.ok).toBe(false);
+    expect(r.issues.some(i => i.queryId === 'q-0001' && i.field === 'text')).toBe(true);
+    expect(r.issues.some(i => i.queryId === 'q-0002' && i.field === 'as_of_date')).toBe(true);
+  });
+});
+
+describe('TEMPORAL_VERBS regex', () => {
+  test('matches common temporal verbs', () => {
+    expect(TEMPORAL_VERBS.test('Where is Sarah?')).toBe(true);
+    expect(TEMPORAL_VERBS.test('Where was Sarah?')).toBe(true);
+    expect(TEMPORAL_VERBS.test('Who were the founders?')).toBe(true);
+    expect(TEMPORAL_VERBS.test('What was the current valuation?')).toBe(true);
+    expect(TEMPORAL_VERBS.test('Where is she now?')).toBe(true);
+    expect(TEMPORAL_VERBS.test('At the time, who led the round?')).toBe(true);
+    expect(TEMPORAL_VERBS.test('During Q1, which deals closed?')).toBe(true);
+    expect(TEMPORAL_VERBS.test('As of 2024, who works at Acme?')).toBe(true);
+    expect(TEMPORAL_VERBS.test('When did Alice join?')).toBe(true);
+  });
+
+  test('does not match non-temporal verbs', () => {
+    expect(TEMPORAL_VERBS.test('Who founded Acme?')).toBe(false);
+    expect(TEMPORAL_VERBS.test('Which investors backed Beta?')).toBe(false);
+    expect(TEMPORAL_VERBS.test('List the advisors at Gamma.')).toBe(false);
+  });
+});

eval/runner/queries/validator.ts

@@ -0,0 +1,204 @@
+/**
+ * Runtime Query schema validator.
+ *
+ * Per the v1.1 eng pass 2 spec. Hand-rolled (no zod dep) to match existing
+ * gbrain codebase style (see src/core/yaml-lite.ts for precedent).
+ *
+ * Validates:
+ *   - Required fields (id, tier, text, expected_output_type, gold)
+ *   - Tier enum
+ *   - expected_output_type enum
+ *   - Temporal `as_of_date` rule: any query with a temporal verb MUST
+ *     set as_of_date to ISO-8601 | "corpus-end" | "per-source"
+ *   - id uniqueness within a batch
+ *   - gold.relevant structure when the expected_output_type is
+ *     'cited-source-pages' (most common tier-1/2/3 pattern)
+ *
+ * Public functions:
+ *   validateQuery(q)      -> ValidationResult single-query
+ *   validateQuerySet(qs)  -> ValidationResult<batch>
+ *
+ * On failure, returns human-readable reasons with the offending query id
+ * so `eval:query:validate` can point contributors at the exact problem.
+ */
+
+import type { Query, Tier, ExpectedOutputType } from '../types.ts';
+
+// ─── Enums ─────────────────────────────────────────────────────────
+
+const VALID_TIERS: readonly Tier[] = [
+  'easy', 'medium', 'hard', 'adversarial', 'fuzzy', 'externally-authored',
+] as const;
+
+const VALID_OUTPUT_TYPES: readonly ExpectedOutputType[] = [
+  'answer-string',
+  'canonical-entity-id',
+  'cited-source-pages',
+  'time-qualified-answer',
+  'abstention',
+  'contradiction-explanation',
+  'poison-flag',
+  'confidence-score',
+] as const;
+
+// ─── Temporal rule (per eng pass 2) ────────────────────────────────
+
+/**
+ * Regex for detecting temporal verbs in query text. If any of these
+ * appear, the query is temporal and MUST carry an `as_of_date` field.
+ * Without that, scoring is ambiguous (which version of the fact is
+ * considered correct?).
+ */
+export const TEMPORAL_VERBS =
+  /\b(is|was|were|current|now|at the time|during|as of|when did)\b/i;
+
+const ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}(T.*)?$/;
+
+// ─── Types ─────────────────────────────────────────────────────────
+
+export interface ValidationIssue {
+  queryId: string;
+  field: string;
+  reason: string;
+}
+
+export interface ValidationResult {
+  ok: boolean;
+  issues: ValidationIssue[];
+  /** Count of queries processed (for batch). 1 for single-query validation. */
+  total: number;
+}
+
+// ─── Individual query validation ───────────────────────────────────
+
+export function validateQuery(q: Query): ValidationResult {
+  const issues: ValidationIssue[] = [];
+  const qid = q.id || '(missing id)';
+
+  if (!q.id || typeof q.id !== 'string' || q.id.trim().length === 0) {
+    issues.push({ queryId: qid, field: 'id', reason: 'id must be a non-empty string (e.g. "q-0001")' });
+  }
+  if (!q.text || typeof q.text !== 'string' || q.text.trim().length === 0) {
+    issues.push({ queryId: qid, field: 'text', reason: 'text must be a non-empty string' });
+  }
+  if (!VALID_TIERS.includes(q.tier)) {
+    issues.push({ queryId: qid, field: 'tier', reason: `tier must be one of ${VALID_TIERS.join(', ')}` });
+  }
+  if (!VALID_OUTPUT_TYPES.includes(q.expected_output_type)) {
+    issues.push({
+      queryId: qid,
+      field: 'expected_output_type',
+      reason: `expected_output_type must be one of ${VALID_OUTPUT_TYPES.join(', ')}`,
+    });
+  }
+  if (!q.gold || typeof q.gold !== 'object') {
+    issues.push({ queryId: qid, field: 'gold', reason: 'gold must be an object' });
+  }
+
+  // Temporal as-of-date rule (eng pass 2).
+  if (q.text && TEMPORAL_VERBS.test(q.text)) {
+    if (q.as_of_date === undefined || q.as_of_date === null || q.as_of_date === '') {
+      issues.push({
+        queryId: qid,
+        field: 'as_of_date',
+        reason:
+          'temporal verb detected; as_of_date required. Set to "corpus-end", "per-source", or an ISO-8601 date.',
+      });
+    } else if (
+      q.as_of_date !== 'corpus-end' &&
+      q.as_of_date !== 'per-source' &&
+      !ISO_DATE_RE.test(q.as_of_date)
+    ) {
+      issues.push({
+        queryId: qid,
+        field: 'as_of_date',
+        reason: 'as_of_date must be "corpus-end", "per-source", or ISO-8601 (YYYY-MM-DD)',
+      });
+    }
+  }
+
+  // If expected_output_type is cited-source-pages, gold.relevant should exist
+  // and be a non-empty array of slug-like strings.
+  if (q.expected_output_type === 'cited-source-pages') {
+    const rel = (q.gold as Record<string, unknown>)?.relevant;
+    if (!Array.isArray(rel) || rel.length === 0) {
+      issues.push({
+        queryId: qid,
+        field: 'gold.relevant',
+        reason: 'cited-source-pages queries require gold.relevant[] with at least one slug',
+      });
+    } else {
+      for (const s of rel) {
+        if (typeof s !== 'string' || !/^[a-z][a-z0-9-]*\/[a-z0-9][a-z0-9-]*$/.test(s)) {
+          issues.push({
+            queryId: qid,
+            field: 'gold.relevant',
+            reason: `slug "${s}" does not match "dir/slug" format (e.g. "people/alice-chen")`,
+          });
+          break; // one message per query is enough
+        }
+      }
+    }
+  }
+
+  // Abstention queries MUST set expected_abstention to true.
+  if (q.expected_output_type === 'abstention') {
+    const expAb = (q.gold as Record<string, unknown>)?.expected_abstention;
+    if (expAb !== true) {
+      issues.push({
+        queryId: qid,
+        field: 'gold.expected_abstention',
+        reason: 'abstention queries require gold.expected_abstention === true',
+      });
+    }
+  }
+
+  // Tier 5.5 externally-authored queries must carry an author field.
+  if (q.tier === 'externally-authored') {
+    if (!q.author || typeof q.author !== 'string' || q.author.trim().length === 0) {
+      issues.push({
+        queryId: qid,
+        field: 'author',
+        reason: 'externally-authored queries require an author field (e.g. "@alice-researcher" or "synthetic-outsider-v1")',
+      });
+    }
+  }
+
+  return { ok: issues.length === 0, issues, total: 1 };
+}
+
+// ─── Batch validation ───────────────────────────────────────────────
+
+export function validateQuerySet(queries: Query[]): ValidationResult {
+  const issues: ValidationIssue[] = [];
+  const seenIds = new Set<string>();
+
+  for (const q of queries) {
+    const r = validateQuery(q);
+    issues.push(...r.issues);
+
+    // Duplicate ID check (batch-level).
+    if (q.id) {
+      if (seenIds.has(q.id)) {
+        issues.push({ queryId: q.id, field: 'id', reason: 'duplicate id in batch' });
+      }
+      seenIds.add(q.id);
+    }
+  }
+
+  return { ok: issues.length === 0, issues, total: queries.length };
+}
+
+// ─── Formatting helpers (for CLI output) ───────────────────────────
+
+export function formatIssues(result: ValidationResult): string {
+  if (result.ok) {
+    return `\u2713 All ${result.total} queries valid.`;
+  }
+  const lines: string[] = [];
+  lines.push(`\u2717 ${result.issues.length} issue(s) across ${result.total} query/queries:`);
+  for (const issue of result.issues) {
+    lines.push(`  [${issue.queryId}] ${issue.field}: ${issue.reason}`);
+  }
+  return lines.join('\n');
+}