diff --git a/src/eval/retrieval-quality/harness.ts b/src/eval/retrieval-quality/harness.ts index e39b81452..780bfb1f8 100644 --- a/src/eval/retrieval-quality/harness.ts +++ b/src/eval/retrieval-quality/harness.ts @@ -33,6 +33,15 @@ export interface NamedThingQuestion { /** slugs that must NOT appear in top-k (hard-negative family). */ forbidden?: string[]; notes?: string; + /** + * v0.43 relational metadata (graph-relationship family). Optional, typed + * (not buried in `notes`) so the relational parser/path/determinism tests + * can machine-check the parse. `seed` is the entity the answer relates to; + * `linkTypes` the edge(s) expected; `kind` the archetype. + */ + seed?: string; + linkTypes?: string[]; + kind?: 'who_rel' | 'who_at' | 'connects' | 'intro'; } /** Ranked slugs for a query, best-first. */ @@ -46,6 +55,11 @@ export interface QuestionResult { reciprocal_rank: number; // 0 if no relevant slug in results /** hard-negative only: true when NO forbidden slug appeared in top-3. */ negative_clean?: boolean; + /** v0.43 — fraction of relevant slugs found in top-K (K=3). 0 for hard-negative. */ + recall_at_k: number; + /** v0.43 — fraction of relevant slugs found in top-10. The relational + * headline metric (a relationship query often has several valid answers). */ + recall_at_10: number; } export interface FamilyReport { @@ -54,6 +68,9 @@ export interface FamilyReport { hit_at_1: number; // rate hit_at_3: number; // rate mrr: number; + /** v0.43 — mean recall@K / recall@10 across the family's questions. */ + recall_at_k: number; + recall_at_10: number; } export interface RetrievalQualityReport { @@ -71,14 +88,20 @@ export function scoreQuestion(q: NamedThingQuestion, ranked: string[]): Question const forbidden = new Set(q.forbidden ?? []); const topK = ranked.slice(0, K); const clean = !topK.some(s => forbidden.has(s)); - return { family: q.family, query: q.query, hit_at_1: clean, hit_at_3: clean, reciprocal_rank: clean ? 1 : 0, negative_clean: clean }; + return { family: q.family, query: q.query, hit_at_1: clean, hit_at_3: clean, reciprocal_rank: clean ? 1 : 0, negative_clean: clean, recall_at_k: 0, recall_at_10: 0 }; } const relevant = new Set(q.relevant ?? []); const firstRelevantIdx = ranked.findIndex(s => relevant.has(s)); const hit1 = firstRelevantIdx === 0; const hit3 = firstRelevantIdx >= 0 && firstRelevantIdx < K; const rr = firstRelevantIdx >= 0 ? 1 / (firstRelevantIdx + 1) : 0; - return { family: q.family, query: q.query, hit_at_1: hit1, hit_at_3: hit3, reciprocal_rank: rr }; + const recallAt = (k: number): number => { + if (relevant.size === 0) return 0; + const top = ranked.slice(0, k); + const found = top.filter(s => relevant.has(s)).length; + return found / relevant.size; + }; + return { family: q.family, query: q.query, hit_at_1: hit1, hit_at_3: hit3, reciprocal_rank: rr, recall_at_k: recallAt(K), recall_at_10: recallAt(10) }; } export async function runRetrievalQuality( @@ -106,6 +129,8 @@ export async function runRetrievalQuality( hit_at_1: n ? list.filter(r => r.hit_at_1).length / n : 0, hit_at_3: n ? list.filter(r => r.hit_at_3).length / n : 0, mrr: n ? list.reduce((s, r) => s + r.reciprocal_rank, 0) / n : 0, + recall_at_k: n ? list.reduce((s, r) => s + r.recall_at_k, 0) / n : 0, + recall_at_10: n ? list.reduce((s, r) => s + r.recall_at_10, 0) / n : 0, }); } families.sort((a, b) => a.family.localeCompare(b.family)); diff --git a/test/fixtures/retrieval-quality/relational/corpus.ts b/test/fixtures/retrieval-quality/relational/corpus.ts new file mode 100644 index 000000000..01354e61d --- /dev/null +++ b/test/fixtures/retrieval-quality/relational/corpus.ts @@ -0,0 +1,176 @@ +/** + * Relational benchmark corpus (v0.43). + * + * A small entity graph whose relational answers are LEXICALLY UNRECOVERABLE: + * every page body is generic and NEVER names the entity it relates to. Only + * the typed edge connects them, so keyword + vector retrieval cannot surface + * the answer — isolating the contribution of the typed-edge recall arm. + * + * Canonical source for BOTH the seed loader and the question set, so the + * corpus and the gold answers can't drift. `relational.jsonl` is generated + * from RELATIONAL_QUESTIONS (a drift test pins them equal). + */ + +import type { BrainEngine } from '../../../../src/core/engine.ts'; +import type { ChunkInput } from '../../../../src/core/types.ts'; +import type { NamedThingQuestion } from '../../../../src/eval/retrieval-quality/harness.ts'; + +// edge maps: target entity → entities related to it via that edge type. +// invested_in / led_round point INTO the company (people/funds → company). +const INVESTMENTS: Record = { + 'companies/widget-co': ['people/alice-example', 'people/bob-example', 'funds/fund-a'], + 'companies/acme-co': ['people/carol-example', 'people/alice-example', 'funds/fund-b'], + 'companies/novapay': ['people/bob-example', 'funds/fund-a'], + 'companies/mindbridge': ['people/dave-example', 'people/carol-example'], + 'companies/helio': ['people/erin-example', 'funds/fund-b'], + 'companies/quanta': ['people/frank-example', 'people/alice-example'], + 'companies/zenith': ['people/grace-example', 'people/heidi-example'], + 'companies/orbital': ['people/ivan-example', 'people/alice-example'], +}; +const EMPLOYMENT: Record = { + 'companies/widget-co': ['people/erin-example', 'people/grace-example'], + 'companies/acme-co': ['people/dave-example', 'people/frank-example'], + 'companies/novapay': ['people/grace-example'], + 'companies/helio': ['people/heidi-example'], + 'companies/zenith': ['people/ivan-example'], +}; +const FOUNDED: Record = { + 'companies/mindbridge': ['people/carol-example'], + 'companies/quanta': ['people/frank-example'], + 'companies/widget-co': ['people/ivan-example'], + 'companies/orbital': ['people/grace-example'], +}; +const ADVISES: Record = { + 'companies/novapay': ['people/frank-example'], + 'companies/helio': ['people/alice-example'], +}; + +// Generic, cross-mention-free bodies keyed by slug prefix. +function bodyFor(slug: string): string { + if (slug.startsWith('companies/')) return 'A privately held company operating in its sector. Founded some years ago; details are sparse in this note.'; + if (slug.startsWith('funds/')) return 'An early-stage venture fund. Writes first checks; portfolio is not enumerated here.'; + return 'An individual in the network. Background and current focus are not described in this note.'; +} + +function titleFor(slug: string): string { + const tail = slug.split('/')[1]; + return tail.replace(/-/g, ' ').replace(/\b\w/g, c => c.toUpperCase()); +} + +/** Every distinct slug in the graph. */ +function allSlugs(): string[] { + const set = new Set(); + for (const map of [INVESTMENTS, EMPLOYMENT, FOUNDED, ADVISES]) { + for (const [company, members] of Object.entries(map)) { + set.add(company); + for (const m of members) set.add(m); + } + } + return [...set].sort(); +} + +function pageType(slug: string): 'company' | 'person' { + // funds + people render as person-ish entity pages; companies as company. + return slug.startsWith('companies/') ? 'company' : 'person'; +} + +// Deterministic per-slug basis embedding so pages are properly indexed +// (chunked) like a real brain. The query has no embedding in CI (no API key), +// so vector search can't connect entities anyway — but the pages must carry a +// chunk to be searchable at all. Body text stays generic + cross-mention-free. +function basisEmbedding(slug: string, dim = 1536): Float32Array { + let h = 0; + for (let i = 0; i < slug.length; i++) h = (h * 31 + slug.charCodeAt(i)) >>> 0; + const e = new Float32Array(dim); + e[h % dim] = 1.0; + return e; +} + +/** Seed the corpus into a fresh brain. Bodies never name related entities. */ +export async function seedRelationalCorpus(engine: BrainEngine): Promise { + for (const slug of allSlugs()) { + const body = bodyFor(slug); + await engine.putPage(slug, { + type: pageType(slug), + title: titleFor(slug), + compiled_truth: body, + timeline: '', + }); + await engine.upsertChunks(slug, [{ + chunk_index: 0, + chunk_text: body, + chunk_source: 'compiled_truth', + embedding: basisEmbedding(slug), + token_count: body.split(/\s+/).length, + }] satisfies ChunkInput[]); + } + const addAll = async (map: Record, linkType: string) => { + for (const [company, members] of Object.entries(map)) { + for (const m of members) { + // edge points member → company (member invested_in / works_at / founded / advises company) + await engine.addLink(m, company, '', linkType, 'manual'); + } + } + }; + await addAll(INVESTMENTS, 'invested_in'); + await addAll(EMPLOYMENT, 'works_at'); + await addAll(FOUNDED, 'founded'); + await addAll(ADVISES, 'advises'); +} + +function bareName(slug: string): string { + return slug.split('/')[1]; +} + +/** Generate the gold question set from the same maps used to seed. */ +export function buildRelationalQuestions(): NamedThingQuestion[] { + const qs: NamedThingQuestion[] = []; + + for (const [company, investors] of Object.entries(INVESTMENTS)) { + qs.push({ + family: 'graph-relationship', kind: 'who_rel', seed: company, linkTypes: ['invested_in', 'led_round'], + query: `who invested in ${bareName(company)}`, relevant: investors, + }); + } + for (const [company, employees] of Object.entries(EMPLOYMENT)) { + qs.push({ + family: 'graph-relationship', kind: 'who_rel', seed: company, linkTypes: ['works_at'], + query: `who works at ${bareName(company)}`, relevant: employees, + }); + } + for (const [company, founders] of Object.entries(FOUNDED)) { + qs.push({ + family: 'graph-relationship', kind: 'who_rel', seed: company, linkTypes: ['founded'], + query: `who founded ${bareName(company)}`, relevant: founders, + }); + } + for (const [company, advisors] of Object.entries(ADVISES)) { + qs.push({ + family: 'graph-relationship', kind: 'who_rel', seed: company, linkTypes: ['advises'], + query: `who advises ${bareName(company)}`, relevant: advisors, + }); + } + + // connects: company pairs that share at least one related entity (any edge). + const relatedTo = (company: string): Set => { + const s = new Set(); + for (const map of [INVESTMENTS, EMPLOYMENT, FOUNDED, ADVISES]) for (const m of map[company] ?? []) s.add(m); + return s; + }; + const companies = Object.keys(INVESTMENTS); + for (let i = 0; i < companies.length; i++) { + for (let j = i + 1; j < companies.length; j++) { + const a = companies[i], b = companies[j]; + const shared = [...relatedTo(a)].filter(x => relatedTo(b).has(x)); + if (shared.length === 0) continue; + qs.push({ + family: 'graph-relationship', kind: 'connects', seed: `${a} ↔ ${b}`, linkTypes: undefined, + query: `what connects ${bareName(a)} and ${bareName(b)}`, relevant: shared.sort(), + }); + } + } + + return qs; +} + +export const RELATIONAL_QUESTIONS: NamedThingQuestion[] = buildRelationalQuestions(); diff --git a/test/fixtures/retrieval-quality/relational/relational.jsonl b/test/fixtures/retrieval-quality/relational/relational.jsonl new file mode 100644 index 000000000..06a4bf28f --- /dev/null +++ b/test/fixtures/retrieval-quality/relational/relational.jsonl @@ -0,0 +1,42 @@ +// Relational benchmark — graph-relationship family. GENERATED from +// test/fixtures/retrieval-quality/relational/corpus.ts (canonical source). +// Answers are lexically unrecoverable — only the typed edge connects +// query to answer. Regenerate: bun run scripts/_gen_relational_jsonl.ts +{"family":"graph-relationship","kind":"who_rel","seed":"companies/widget-co","linkTypes":["invested_in","led_round"],"query":"who invested in widget-co","relevant":["people/alice-example","people/bob-example","funds/fund-a"]} +{"family":"graph-relationship","kind":"who_rel","seed":"companies/acme-co","linkTypes":["invested_in","led_round"],"query":"who invested in acme-co","relevant":["people/carol-example","people/alice-example","funds/fund-b"]} +{"family":"graph-relationship","kind":"who_rel","seed":"companies/novapay","linkTypes":["invested_in","led_round"],"query":"who invested in novapay","relevant":["people/bob-example","funds/fund-a"]} +{"family":"graph-relationship","kind":"who_rel","seed":"companies/mindbridge","linkTypes":["invested_in","led_round"],"query":"who invested in mindbridge","relevant":["people/dave-example","people/carol-example"]} +{"family":"graph-relationship","kind":"who_rel","seed":"companies/helio","linkTypes":["invested_in","led_round"],"query":"who invested in helio","relevant":["people/erin-example","funds/fund-b"]} +{"family":"graph-relationship","kind":"who_rel","seed":"companies/quanta","linkTypes":["invested_in","led_round"],"query":"who invested in quanta","relevant":["people/frank-example","people/alice-example"]} +{"family":"graph-relationship","kind":"who_rel","seed":"companies/zenith","linkTypes":["invested_in","led_round"],"query":"who invested in zenith","relevant":["people/grace-example","people/heidi-example"]} +{"family":"graph-relationship","kind":"who_rel","seed":"companies/orbital","linkTypes":["invested_in","led_round"],"query":"who invested in orbital","relevant":["people/ivan-example","people/alice-example"]} +{"family":"graph-relationship","kind":"who_rel","seed":"companies/widget-co","linkTypes":["works_at"],"query":"who works at widget-co","relevant":["people/erin-example","people/grace-example"]} +{"family":"graph-relationship","kind":"who_rel","seed":"companies/acme-co","linkTypes":["works_at"],"query":"who works at acme-co","relevant":["people/dave-example","people/frank-example"]} +{"family":"graph-relationship","kind":"who_rel","seed":"companies/novapay","linkTypes":["works_at"],"query":"who works at novapay","relevant":["people/grace-example"]} +{"family":"graph-relationship","kind":"who_rel","seed":"companies/helio","linkTypes":["works_at"],"query":"who works at helio","relevant":["people/heidi-example"]} +{"family":"graph-relationship","kind":"who_rel","seed":"companies/zenith","linkTypes":["works_at"],"query":"who works at zenith","relevant":["people/ivan-example"]} +{"family":"graph-relationship","kind":"who_rel","seed":"companies/mindbridge","linkTypes":["founded"],"query":"who founded mindbridge","relevant":["people/carol-example"]} +{"family":"graph-relationship","kind":"who_rel","seed":"companies/quanta","linkTypes":["founded"],"query":"who founded quanta","relevant":["people/frank-example"]} +{"family":"graph-relationship","kind":"who_rel","seed":"companies/widget-co","linkTypes":["founded"],"query":"who founded widget-co","relevant":["people/ivan-example"]} +{"family":"graph-relationship","kind":"who_rel","seed":"companies/orbital","linkTypes":["founded"],"query":"who founded orbital","relevant":["people/grace-example"]} +{"family":"graph-relationship","kind":"who_rel","seed":"companies/novapay","linkTypes":["advises"],"query":"who advises novapay","relevant":["people/frank-example"]} +{"family":"graph-relationship","kind":"who_rel","seed":"companies/helio","linkTypes":["advises"],"query":"who advises helio","relevant":["people/alice-example"]} +{"family":"graph-relationship","kind":"connects","seed":"companies/widget-co ↔ companies/acme-co","query":"what connects widget-co and acme-co","relevant":["people/alice-example"]} +{"family":"graph-relationship","kind":"connects","seed":"companies/widget-co ↔ companies/novapay","query":"what connects widget-co and novapay","relevant":["funds/fund-a","people/bob-example","people/grace-example"]} +{"family":"graph-relationship","kind":"connects","seed":"companies/widget-co ↔ companies/helio","query":"what connects widget-co and helio","relevant":["people/alice-example","people/erin-example"]} +{"family":"graph-relationship","kind":"connects","seed":"companies/widget-co ↔ companies/quanta","query":"what connects widget-co and quanta","relevant":["people/alice-example"]} +{"family":"graph-relationship","kind":"connects","seed":"companies/widget-co ↔ companies/zenith","query":"what connects widget-co and zenith","relevant":["people/grace-example","people/ivan-example"]} +{"family":"graph-relationship","kind":"connects","seed":"companies/widget-co ↔ companies/orbital","query":"what connects widget-co and orbital","relevant":["people/alice-example","people/grace-example","people/ivan-example"]} +{"family":"graph-relationship","kind":"connects","seed":"companies/acme-co ↔ companies/novapay","query":"what connects acme-co and novapay","relevant":["people/frank-example"]} +{"family":"graph-relationship","kind":"connects","seed":"companies/acme-co ↔ companies/mindbridge","query":"what connects acme-co and mindbridge","relevant":["people/carol-example","people/dave-example"]} +{"family":"graph-relationship","kind":"connects","seed":"companies/acme-co ↔ companies/helio","query":"what connects acme-co and helio","relevant":["funds/fund-b","people/alice-example"]} +{"family":"graph-relationship","kind":"connects","seed":"companies/acme-co ↔ companies/quanta","query":"what connects acme-co and quanta","relevant":["people/alice-example","people/frank-example"]} +{"family":"graph-relationship","kind":"connects","seed":"companies/acme-co ↔ companies/orbital","query":"what connects acme-co and orbital","relevant":["people/alice-example"]} +{"family":"graph-relationship","kind":"connects","seed":"companies/novapay ↔ companies/quanta","query":"what connects novapay and quanta","relevant":["people/frank-example"]} +{"family":"graph-relationship","kind":"connects","seed":"companies/novapay ↔ companies/zenith","query":"what connects novapay and zenith","relevant":["people/grace-example"]} +{"family":"graph-relationship","kind":"connects","seed":"companies/novapay ↔ companies/orbital","query":"what connects novapay and orbital","relevant":["people/grace-example"]} +{"family":"graph-relationship","kind":"connects","seed":"companies/helio ↔ companies/quanta","query":"what connects helio and quanta","relevant":["people/alice-example"]} +{"family":"graph-relationship","kind":"connects","seed":"companies/helio ↔ companies/zenith","query":"what connects helio and zenith","relevant":["people/heidi-example"]} +{"family":"graph-relationship","kind":"connects","seed":"companies/helio ↔ companies/orbital","query":"what connects helio and orbital","relevant":["people/alice-example"]} +{"family":"graph-relationship","kind":"connects","seed":"companies/quanta ↔ companies/orbital","query":"what connects quanta and orbital","relevant":["people/alice-example"]} +{"family":"graph-relationship","kind":"connects","seed":"companies/zenith ↔ companies/orbital","query":"what connects zenith and orbital","relevant":["people/grace-example","people/ivan-example"]}