feat(eval): relational benchmark + recall@k harness metrics

NamedThingBench harness gains recall@k / recall@10 (the relational headline
metric) on QuestionResult + FamilyReport, plus typed seed/linkTypes/kind on
NamedThingQuestion so the graph-relationship family is machine-checkable.

Adds the relational benchmark corpus (test/fixtures/retrieval-quality/
relational/): a small entity graph whose answers are LEXICALLY UNRECOVERABLE
— every page body is generic and never names the entity it relates to, so
only the typed edge connects query to answer. corpus.ts is the canonical
source for both the seed loader and the 38-question gold set; relational.jsonl
is generated from it (a drift test pins them equal).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

src/eval/retrieval-quality/harness.ts

@@ -33,6 +33,15 @@ export interface NamedThingQuestion {
   /** slugs that must NOT appear in top-k (hard-negative family). */
   forbidden?: string[];
   notes?: string;
+  /**
+   * v0.43 relational metadata (graph-relationship family). Optional, typed
+   * (not buried in `notes`) so the relational parser/path/determinism tests
+   * can machine-check the parse. `seed` is the entity the answer relates to;
+   * `linkTypes` the edge(s) expected; `kind` the archetype.
+   */
+  seed?: string;
+  linkTypes?: string[];
+  kind?: 'who_rel' | 'who_at' | 'connects' | 'intro';
 }
 
 /** Ranked slugs for a query, best-first. */
@@ -46,6 +55,11 @@ export interface QuestionResult {
   reciprocal_rank: number; // 0 if no relevant slug in results
   /** hard-negative only: true when NO forbidden slug appeared in top-3. */
   negative_clean?: boolean;
+  /** v0.43 — fraction of relevant slugs found in top-K (K=3). 0 for hard-negative. */
+  recall_at_k: number;
+  /** v0.43 — fraction of relevant slugs found in top-10. The relational
+   *  headline metric (a relationship query often has several valid answers). */
+  recall_at_10: number;
 }
 
 export interface FamilyReport {
@@ -54,6 +68,9 @@ export interface FamilyReport {
   hit_at_1: number; // rate
   hit_at_3: number; // rate
   mrr: number;
+  /** v0.43 — mean recall@K / recall@10 across the family's questions. */
+  recall_at_k: number;
+  recall_at_10: number;
 }
 
 export interface RetrievalQualityReport {
@@ -71,14 +88,20 @@ export function scoreQuestion(q: NamedThingQuestion, ranked: string[]): Question
     const forbidden = new Set(q.forbidden ?? []);
     const topK = ranked.slice(0, K);
     const clean = !topK.some(s => forbidden.has(s));
-    return { family: q.family, query: q.query, hit_at_1: clean, hit_at_3: clean, reciprocal_rank: clean ? 1 : 0, negative_clean: clean };
+    return { family: q.family, query: q.query, hit_at_1: clean, hit_at_3: clean, reciprocal_rank: clean ? 1 : 0, negative_clean: clean, recall_at_k: 0, recall_at_10: 0 };
   }
   const relevant = new Set(q.relevant ?? []);
   const firstRelevantIdx = ranked.findIndex(s => relevant.has(s));
   const hit1 = firstRelevantIdx === 0;
   const hit3 = firstRelevantIdx >= 0 && firstRelevantIdx < K;
   const rr = firstRelevantIdx >= 0 ? 1 / (firstRelevantIdx + 1) : 0;
-  return { family: q.family, query: q.query, hit_at_1: hit1, hit_at_3: hit3, reciprocal_rank: rr };
+  const recallAt = (k: number): number => {
+    if (relevant.size === 0) return 0;
+    const top = ranked.slice(0, k);
+    const found = top.filter(s => relevant.has(s)).length;
+    return found / relevant.size;
+  };
+  return { family: q.family, query: q.query, hit_at_1: hit1, hit_at_3: hit3, reciprocal_rank: rr, recall_at_k: recallAt(K), recall_at_10: recallAt(10) };
 }
 
 export async function runRetrievalQuality(
@@ -106,6 +129,8 @@ export async function runRetrievalQuality(
       hit_at_1: n ? list.filter(r => r.hit_at_1).length / n : 0,
       hit_at_3: n ? list.filter(r => r.hit_at_3).length / n : 0,
       mrr: n ? list.reduce((s, r) => s + r.reciprocal_rank, 0) / n : 0,
+      recall_at_k: n ? list.reduce((s, r) => s + r.recall_at_k, 0) / n : 0,
+      recall_at_10: n ? list.reduce((s, r) => s + r.recall_at_10, 0) / n : 0,
     });
   }
   families.sort((a, b) => a.family.localeCompare(b.family));

test/fixtures/retrieval-quality/relational/corpus.ts

@@ -0,0 +1,176 @@
+/**
+ * Relational benchmark corpus (v0.43).
+ *
+ * A small entity graph whose relational answers are LEXICALLY UNRECOVERABLE:
+ * every page body is generic and NEVER names the entity it relates to. Only
+ * the typed edge connects them, so keyword + vector retrieval cannot surface
+ * the answer — isolating the contribution of the typed-edge recall arm.
+ *
+ * Canonical source for BOTH the seed loader and the question set, so the
+ * corpus and the gold answers can't drift. `relational.jsonl` is generated
+ * from RELATIONAL_QUESTIONS (a drift test pins them equal).
+ */
+
+import type { BrainEngine } from '../../../../src/core/engine.ts';
+import type { ChunkInput } from '../../../../src/core/types.ts';
+import type { NamedThingQuestion } from '../../../../src/eval/retrieval-quality/harness.ts';
+
+// edge maps: target entity → entities related to it via that edge type.
+// invested_in / led_round point INTO the company (people/funds → company).
+const INVESTMENTS: Record<string, string[]> = {
+  'companies/widget-co': ['people/alice-example', 'people/bob-example', 'funds/fund-a'],
+  'companies/acme-co': ['people/carol-example', 'people/alice-example', 'funds/fund-b'],
+  'companies/novapay': ['people/bob-example', 'funds/fund-a'],
+  'companies/mindbridge': ['people/dave-example', 'people/carol-example'],
+  'companies/helio': ['people/erin-example', 'funds/fund-b'],
+  'companies/quanta': ['people/frank-example', 'people/alice-example'],
+  'companies/zenith': ['people/grace-example', 'people/heidi-example'],
+  'companies/orbital': ['people/ivan-example', 'people/alice-example'],
+};
+const EMPLOYMENT: Record<string, string[]> = {
+  'companies/widget-co': ['people/erin-example', 'people/grace-example'],
+  'companies/acme-co': ['people/dave-example', 'people/frank-example'],
+  'companies/novapay': ['people/grace-example'],
+  'companies/helio': ['people/heidi-example'],
+  'companies/zenith': ['people/ivan-example'],
+};
+const FOUNDED: Record<string, string[]> = {
+  'companies/mindbridge': ['people/carol-example'],
+  'companies/quanta': ['people/frank-example'],
+  'companies/widget-co': ['people/ivan-example'],
+  'companies/orbital': ['people/grace-example'],
+};
+const ADVISES: Record<string, string[]> = {
+  'companies/novapay': ['people/frank-example'],
+  'companies/helio': ['people/alice-example'],
+};
+
+// Generic, cross-mention-free bodies keyed by slug prefix.
+function bodyFor(slug: string): string {
+  if (slug.startsWith('companies/')) return 'A privately held company operating in its sector. Founded some years ago; details are sparse in this note.';
+  if (slug.startsWith('funds/')) return 'An early-stage venture fund. Writes first checks; portfolio is not enumerated here.';
+  return 'An individual in the network. Background and current focus are not described in this note.';
+}
+
+function titleFor(slug: string): string {
+  const tail = slug.split('/')[1];
+  return tail.replace(/-/g, ' ').replace(/\b\w/g, c => c.toUpperCase());
+}
+
+/** Every distinct slug in the graph. */
+function allSlugs(): string[] {
+  const set = new Set<string>();
+  for (const map of [INVESTMENTS, EMPLOYMENT, FOUNDED, ADVISES]) {
+    for (const [company, members] of Object.entries(map)) {
+      set.add(company);
+      for (const m of members) set.add(m);
+    }
+  }
+  return [...set].sort();
+}
+
+function pageType(slug: string): 'company' | 'person' {
+  // funds + people render as person-ish entity pages; companies as company.
+  return slug.startsWith('companies/') ? 'company' : 'person';
+}
+
+// Deterministic per-slug basis embedding so pages are properly indexed
+// (chunked) like a real brain. The query has no embedding in CI (no API key),
+// so vector search can't connect entities anyway — but the pages must carry a
+// chunk to be searchable at all. Body text stays generic + cross-mention-free.
+function basisEmbedding(slug: string, dim = 1536): Float32Array {
+  let h = 0;
+  for (let i = 0; i < slug.length; i++) h = (h * 31 + slug.charCodeAt(i)) >>> 0;
+  const e = new Float32Array(dim);
+  e[h % dim] = 1.0;
+  return e;
+}
+
+/** Seed the corpus into a fresh brain. Bodies never name related entities. */
+export async function seedRelationalCorpus(engine: BrainEngine): Promise<void> {
+  for (const slug of allSlugs()) {
+    const body = bodyFor(slug);
+    await engine.putPage(slug, {
+      type: pageType(slug),
+      title: titleFor(slug),
+      compiled_truth: body,
+      timeline: '',
+    });
+    await engine.upsertChunks(slug, [{
+      chunk_index: 0,
+      chunk_text: body,
+      chunk_source: 'compiled_truth',
+      embedding: basisEmbedding(slug),
+      token_count: body.split(/\s+/).length,
+    }] satisfies ChunkInput[]);
+  }
+  const addAll = async (map: Record<string, string[]>, linkType: string) => {
+    for (const [company, members] of Object.entries(map)) {
+      for (const m of members) {
+        // edge points member → company (member invested_in / works_at / founded / advises company)
+        await engine.addLink(m, company, '', linkType, 'manual');
+      }
+    }
+  };
+  await addAll(INVESTMENTS, 'invested_in');
+  await addAll(EMPLOYMENT, 'works_at');
+  await addAll(FOUNDED, 'founded');
+  await addAll(ADVISES, 'advises');
+}
+
+function bareName(slug: string): string {
+  return slug.split('/')[1];
+}
+
+/** Generate the gold question set from the same maps used to seed. */
+export function buildRelationalQuestions(): NamedThingQuestion[] {
+  const qs: NamedThingQuestion[] = [];
+
+  for (const [company, investors] of Object.entries(INVESTMENTS)) {
+    qs.push({
+      family: 'graph-relationship', kind: 'who_rel', seed: company, linkTypes: ['invested_in', 'led_round'],
+      query: `who invested in ${bareName(company)}`, relevant: investors,
+    });
+  }
+  for (const [company, employees] of Object.entries(EMPLOYMENT)) {
+    qs.push({
+      family: 'graph-relationship', kind: 'who_rel', seed: company, linkTypes: ['works_at'],
+      query: `who works at ${bareName(company)}`, relevant: employees,
+    });
+  }
+  for (const [company, founders] of Object.entries(FOUNDED)) {
+    qs.push({
+      family: 'graph-relationship', kind: 'who_rel', seed: company, linkTypes: ['founded'],
+      query: `who founded ${bareName(company)}`, relevant: founders,
+    });
+  }
+  for (const [company, advisors] of Object.entries(ADVISES)) {
+    qs.push({
+      family: 'graph-relationship', kind: 'who_rel', seed: company, linkTypes: ['advises'],
+      query: `who advises ${bareName(company)}`, relevant: advisors,
+    });
+  }
+
+  // connects: company pairs that share at least one related entity (any edge).
+  const relatedTo = (company: string): Set<string> => {
+    const s = new Set<string>();
+    for (const map of [INVESTMENTS, EMPLOYMENT, FOUNDED, ADVISES]) for (const m of map[company] ?? []) s.add(m);
+    return s;
+  };
+  const companies = Object.keys(INVESTMENTS);
+  for (let i = 0; i < companies.length; i++) {
+    for (let j = i + 1; j < companies.length; j++) {
+      const a = companies[i], b = companies[j];
+      const shared = [...relatedTo(a)].filter(x => relatedTo(b).has(x));
+      if (shared.length === 0) continue;
+      qs.push({
+        family: 'graph-relationship', kind: 'connects', seed: `${a} ↔ ${b}`, linkTypes: undefined,
+        query: `what connects ${bareName(a)} and ${bareName(b)}`, relevant: shared.sort(),
+      });
+    }
+  }
+
+  return qs;
+}
+
+export const RELATIONAL_QUESTIONS: NamedThingQuestion[] = buildRelationalQuestions();

test/fixtures/retrieval-quality/relational/relational.jsonl

@@ -0,0 +1,42 @@
+// Relational benchmark — graph-relationship family. GENERATED from
+// test/fixtures/retrieval-quality/relational/corpus.ts (canonical source).
+// Answers are lexically unrecoverable — only the typed edge connects
+// query to answer. Regenerate: bun run scripts/_gen_relational_jsonl.ts
+{"family":"graph-relationship","kind":"who_rel","seed":"companies/widget-co","linkTypes":["invested_in","led_round"],"query":"who invested in widget-co","relevant":["people/alice-example","people/bob-example","funds/fund-a"]}
+{"family":"graph-relationship","kind":"who_rel","seed":"companies/acme-co","linkTypes":["invested_in","led_round"],"query":"who invested in acme-co","relevant":["people/carol-example","people/alice-example","funds/fund-b"]}
+{"family":"graph-relationship","kind":"who_rel","seed":"companies/novapay","linkTypes":["invested_in","led_round"],"query":"who invested in novapay","relevant":["people/bob-example","funds/fund-a"]}
+{"family":"graph-relationship","kind":"who_rel","seed":"companies/mindbridge","linkTypes":["invested_in","led_round"],"query":"who invested in mindbridge","relevant":["people/dave-example","people/carol-example"]}
+{"family":"graph-relationship","kind":"who_rel","seed":"companies/helio","linkTypes":["invested_in","led_round"],"query":"who invested in helio","relevant":["people/erin-example","funds/fund-b"]}
+{"family":"graph-relationship","kind":"who_rel","seed":"companies/quanta","linkTypes":["invested_in","led_round"],"query":"who invested in quanta","relevant":["people/frank-example","people/alice-example"]}
+{"family":"graph-relationship","kind":"who_rel","seed":"companies/zenith","linkTypes":["invested_in","led_round"],"query":"who invested in zenith","relevant":["people/grace-example","people/heidi-example"]}
+{"family":"graph-relationship","kind":"who_rel","seed":"companies/orbital","linkTypes":["invested_in","led_round"],"query":"who invested in orbital","relevant":["people/ivan-example","people/alice-example"]}
+{"family":"graph-relationship","kind":"who_rel","seed":"companies/widget-co","linkTypes":["works_at"],"query":"who works at widget-co","relevant":["people/erin-example","people/grace-example"]}
+{"family":"graph-relationship","kind":"who_rel","seed":"companies/acme-co","linkTypes":["works_at"],"query":"who works at acme-co","relevant":["people/dave-example","people/frank-example"]}
+{"family":"graph-relationship","kind":"who_rel","seed":"companies/novapay","linkTypes":["works_at"],"query":"who works at novapay","relevant":["people/grace-example"]}
+{"family":"graph-relationship","kind":"who_rel","seed":"companies/helio","linkTypes":["works_at"],"query":"who works at helio","relevant":["people/heidi-example"]}
+{"family":"graph-relationship","kind":"who_rel","seed":"companies/zenith","linkTypes":["works_at"],"query":"who works at zenith","relevant":["people/ivan-example"]}
+{"family":"graph-relationship","kind":"who_rel","seed":"companies/mindbridge","linkTypes":["founded"],"query":"who founded mindbridge","relevant":["people/carol-example"]}
+{"family":"graph-relationship","kind":"who_rel","seed":"companies/quanta","linkTypes":["founded"],"query":"who founded quanta","relevant":["people/frank-example"]}
+{"family":"graph-relationship","kind":"who_rel","seed":"companies/widget-co","linkTypes":["founded"],"query":"who founded widget-co","relevant":["people/ivan-example"]}
+{"family":"graph-relationship","kind":"who_rel","seed":"companies/orbital","linkTypes":["founded"],"query":"who founded orbital","relevant":["people/grace-example"]}
+{"family":"graph-relationship","kind":"who_rel","seed":"companies/novapay","linkTypes":["advises"],"query":"who advises novapay","relevant":["people/frank-example"]}
+{"family":"graph-relationship","kind":"who_rel","seed":"companies/helio","linkTypes":["advises"],"query":"who advises helio","relevant":["people/alice-example"]}
+{"family":"graph-relationship","kind":"connects","seed":"companies/widget-co ↔ companies/acme-co","query":"what connects widget-co and acme-co","relevant":["people/alice-example"]}
+{"family":"graph-relationship","kind":"connects","seed":"companies/widget-co ↔ companies/novapay","query":"what connects widget-co and novapay","relevant":["funds/fund-a","people/bob-example","people/grace-example"]}
+{"family":"graph-relationship","kind":"connects","seed":"companies/widget-co ↔ companies/helio","query":"what connects widget-co and helio","relevant":["people/alice-example","people/erin-example"]}
+{"family":"graph-relationship","kind":"connects","seed":"companies/widget-co ↔ companies/quanta","query":"what connects widget-co and quanta","relevant":["people/alice-example"]}
+{"family":"graph-relationship","kind":"connects","seed":"companies/widget-co ↔ companies/zenith","query":"what connects widget-co and zenith","relevant":["people/grace-example","people/ivan-example"]}
+{"family":"graph-relationship","kind":"connects","seed":"companies/widget-co ↔ companies/orbital","query":"what connects widget-co and orbital","relevant":["people/alice-example","people/grace-example","people/ivan-example"]}
+{"family":"graph-relationship","kind":"connects","seed":"companies/acme-co ↔ companies/novapay","query":"what connects acme-co and novapay","relevant":["people/frank-example"]}
+{"family":"graph-relationship","kind":"connects","seed":"companies/acme-co ↔ companies/mindbridge","query":"what connects acme-co and mindbridge","relevant":["people/carol-example","people/dave-example"]}
+{"family":"graph-relationship","kind":"connects","seed":"companies/acme-co ↔ companies/helio","query":"what connects acme-co and helio","relevant":["funds/fund-b","people/alice-example"]}
+{"family":"graph-relationship","kind":"connects","seed":"companies/acme-co ↔ companies/quanta","query":"what connects acme-co and quanta","relevant":["people/alice-example","people/frank-example"]}
+{"family":"graph-relationship","kind":"connects","seed":"companies/acme-co ↔ companies/orbital","query":"what connects acme-co and orbital","relevant":["people/alice-example"]}
+{"family":"graph-relationship","kind":"connects","seed":"companies/novapay ↔ companies/quanta","query":"what connects novapay and quanta","relevant":["people/frank-example"]}
+{"family":"graph-relationship","kind":"connects","seed":"companies/novapay ↔ companies/zenith","query":"what connects novapay and zenith","relevant":["people/grace-example"]}
+{"family":"graph-relationship","kind":"connects","seed":"companies/novapay ↔ companies/orbital","query":"what connects novapay and orbital","relevant":["people/grace-example"]}
+{"family":"graph-relationship","kind":"connects","seed":"companies/helio ↔ companies/quanta","query":"what connects helio and quanta","relevant":["people/alice-example"]}
+{"family":"graph-relationship","kind":"connects","seed":"companies/helio ↔ companies/zenith","query":"what connects helio and zenith","relevant":["people/heidi-example"]}
+{"family":"graph-relationship","kind":"connects","seed":"companies/helio ↔ companies/orbital","query":"what connects helio and orbital","relevant":["people/alice-example"]}
+{"family":"graph-relationship","kind":"connects","seed":"companies/quanta ↔ companies/orbital","query":"what connects quanta and orbital","relevant":["people/alice-example"]}
+{"family":"graph-relationship","kind":"connects","seed":"companies/zenith ↔ companies/orbital","query":"what connects zenith and orbital","relevant":["people/grace-example","people/ivan-example"]}