diff --git a/src/commands/eval-retrieval-quality.ts b/src/commands/eval-retrieval-quality.ts index 175a1e7a3..7d48dc10f 100644 --- a/src/commands/eval-retrieval-quality.ts +++ b/src/commands/eval-retrieval-quality.ts @@ -20,12 +20,13 @@ import { export async function runEvalRetrievalQuality(engine: BrainEngine, args: string[]): Promise { const json = args.includes('--json'); + const abRelational = args.includes('--ab-relational'); const sourceIdx = args.indexOf('--source'); const sourceId = sourceIdx >= 0 ? args[sourceIdx + 1] : undefined; const fixture = args.find(a => !a.startsWith('--') && a !== sourceId); if (!fixture) { - console.error('Usage: gbrain eval retrieval-quality [--json] [--source ]'); + console.error('Usage: gbrain eval retrieval-quality [--json] [--source ] [--ab-relational]'); process.exit(2); } @@ -47,6 +48,48 @@ export async function runEvalRetrievalQuality(engine: BrainEngine, args: string[ return results.map(r => r.slug); }; + // v0.43 — A/B the relational recall arm (off vs on) over the same questions + // in a fixed mode (expansion off, skipCache-equivalent via bare hybridSearch) + // so the delta is purely the arm. Headline: graph-relationship recall@10 lift. + if (abRelational) { + const mk = (relationalRetrieval: boolean): SearchFn => async (q) => { + const results = await hybridSearch(engine, q, { + limit: 10, relationalRetrieval, expansion: false, + ...(sourceId ? { sourceId } : {}), + }); + return results.map(r => r.slug); + }; + const t0 = Date.now(); + const off = await runRetrievalQuality(questions, mk(false)); + const tMid = Date.now(); + const on = await runRetrievalQuality(questions, mk(true)); + const tEnd = Date.now(); + const fam = (r: typeof off, f: string) => r.families.find(x => x.family === f); + const rel = { off: fam(off, 'graph-relationship'), on: fam(on, 'graph-relationship') }; + const payload = { + schema_version: 1 as const, + ab: 'relational', + graph_relationship: { + n: rel.on?.n ?? 0, + recall_at_10: { off: rel.off?.recall_at_10 ?? 0, on: rel.on?.recall_at_10 ?? 0, + delta: (rel.on?.recall_at_10 ?? 0) - (rel.off?.recall_at_10 ?? 0) }, + hit_at_3: { off: rel.off?.hit_at_3 ?? 0, on: rel.on?.hit_at_3 ?? 0 }, + }, + latency_ms: { off_total: tMid - t0, on_total: tEnd - tMid }, + families: { off: off.families, on: on.families }, + }; + if (json) { + console.log(JSON.stringify(payload, null, 2)); + } else { + console.log(`Relational A/B — graph-relationship n=${payload.graph_relationship.n}`); + const g = payload.graph_relationship; + console.log(` recall@10: off=${(g.recall_at_10.off * 100).toFixed(0)}% on=${(g.recall_at_10.on * 100).toFixed(0)}% Δ=+${(g.recall_at_10.delta * 100).toFixed(0)}pp`); + console.log(` Hit@3: off=${(g.hit_at_3.off * 100).toFixed(0)}% on=${(g.hit_at_3.on * 100).toFixed(0)}%`); + console.log(` latency: off=${payload.latency_ms.off_total}ms on=${payload.latency_ms.on_total}ms (arm adds ${payload.latency_ms.on_total - payload.latency_ms.off_total}ms over ${payload.graph_relationship.n} queries)`); + } + process.exit(0); + } + const report = await runRetrievalQuality(questions, searchFn); const gate = evaluateGate(report); diff --git a/src/core/search/hybrid.ts b/src/core/search/hybrid.ts index f23fd035c..ee1b08c6e 100644 --- a/src/core/search/hybrid.ts +++ b/src/core/search/hybrid.ts @@ -981,6 +981,23 @@ export async function hybridSearch( titleBoost: resolvedMode.title_boost, }; + // v0.43 — build the relational recall arm ONCE here, before any return + // path, so typed-edge answers contribute on ALL THREE paths: the + // no-embedding-provider path, the embed-failed keyword fallback, and the + // main RRF path. Parsed from the original query (deterministic); empty for + // non-relational queries → pure no-op. (Modality gate lives on the main + // path; the parser only matches text-shaped relational queries anyway.) + let relationalList: SearchResult[] = []; + if (resolvedMode.relationalRetrieval) { + relationalList = await buildRelationalArm(engine, query, { + sourceId: opts?.sourceId, + sourceIds: opts?.sourceIds, + depth: resolvedMode.relational_retrieval_depth, + limit: opts?.limit ?? resolvedMode.searchLimit, + onMeta: opts?.onRelationalMeta, + }); + } + // Skip vector search entirely if the gateway has no embedding provider configured (Codex C3). // v0.36 (D10): ask "is the RESOLVED column's provider reachable?" rather // than "is the global default reachable?" — otherwise an unreachable @@ -989,13 +1006,24 @@ export async function hybridSearch( const { isAvailable } = await import('../ai/gateway.ts'); const providerProbe = resolvedCol.embeddingModel || undefined; if (!isAvailable('embedding', providerProbe)) { - if (keywordResults.length > 0) { - await runPostFusionStages(engine, keywordResults, postFusionOpts); - keywordResults.sort((a, b) => b.score - a.score); + // v0.43 — fuse the relational arm with keyword so typed-edge answers + // survive on the no-embedding-provider path (the relational win is most + // valuable exactly when vector is unavailable). + let noEmbedResults = keywordResults; + if (relationalList.length > 0) { + const fk = opts?.rrfK ?? RRF_K; + noEmbedResults = rrfFusionWeighted( + [{ list: keywordResults, k: fk }, { list: relationalList, k: fk }], + detailResolved !== 'high', + ); + } + if (noEmbedResults.length > 0) { + await runPostFusionStages(engine, noEmbedResults, postFusionOpts); + noEmbedResults.sort((a, b) => b.score - a.score); } // T3/T4 — alias hop + evidence stamp even without an embedding provider // (the named-thing fix is most valuable exactly when vector is unavailable). - const noEmbedHopped = await applyAliasHop(engine, dedupResults(keywordResults), query, { + const noEmbedHopped = await applyAliasHop(engine, dedupResults(noEmbedResults), query, { sourceId: opts?.sourceId, sourceIds: opts?.sourceIds, }); @@ -1213,11 +1241,21 @@ export async function hybridSearch( // v0.29.1 codex pass-2 #4: this is the third return path. Apply // post-fusion stages here too — without it, salience='on' silently // does nothing on embed failures. - if (keywordResults.length > 0) { - await runPostFusionStages(engine, keywordResults, postFusionOpts); - keywordResults.sort((a, b) => b.score - a.score); + // v0.43: fuse the relational arm with keyword via RRF so typed-edge + // answers survive even when vector is unavailable. + let fallbackResults = keywordResults; + if (relationalList.length > 0) { + const fk = opts?.rrfK ?? RRF_K; + fallbackResults = rrfFusionWeighted( + [{ list: keywordResults, k: fk }, { list: relationalList, k: fk }], + detail !== 'high', + ); } - const kwHopped = await applyAliasHop(engine, dedupResults(keywordResults), query, { + if (fallbackResults.length > 0) { + await runPostFusionStages(engine, fallbackResults, postFusionOpts); + fallbackResults.sort((a, b) => b.score - a.score); + } + const kwHopped = await applyAliasHop(engine, dedupResults(fallbackResults), query, { sourceId: opts?.sourceId, sourceIds: opts?.sourceIds, }); @@ -1276,25 +1314,13 @@ export async function hybridSearch( { list: keywordResults, k: keywordK }, ]; - // v0.43 — relational recall arm (fourth RRF arm). Fires only when the mode - // enables it AND the query is text-shaped (E4: no-op in image-only mode). - // Parsed from the ORIGINAL query (never expanded variants) so it's - // deterministic. Empty list = pure no-op for non-relational queries. Rides - // every downstream stage (cosine re-score, post-fusion boosts, dedup, - // reranker, autocut, token budget) like any other arm. - if (resolvedMode.relationalRetrieval && effectiveModality !== 'image') { - const relationalList = await buildRelationalArm(engine, query, { - sourceId: opts?.sourceId, - sourceIds: opts?.sourceIds, - depth: resolvedMode.relational_retrieval_depth, - limit: opts?.limit ?? resolvedMode.searchLimit, - onMeta: opts?.onRelationalMeta, - }); - if (relationalList.length > 0) { - // Neutral weight: competes evenly with keyword/vector (baseRrfK), not - // dominating. Calibrated against the Commit 5 A/B. - allLists.push({ list: relationalList, k: baseRrfK }); - } + // v0.43 — relational recall arm (fourth RRF arm), built above so it also + // contributes on the keyword-only fallback path. Neutral weight (baseRrfK): + // competes evenly with keyword/vector, not dominating. Empty for + // non-relational queries → pure no-op. Rides every downstream stage (cosine + // re-score, post-fusion boosts, dedup, reranker, autocut, token budget). + if (relationalList.length > 0 && effectiveModality !== 'image') { + allLists.push({ list: relationalList, k: baseRrfK }); } let fused = rrfFusionWeighted(allLists, detail !== 'high'); diff --git a/test/relational-ab.test.ts b/test/relational-ab.test.ts new file mode 100644 index 000000000..b1c747f32 --- /dev/null +++ b/test/relational-ab.test.ts @@ -0,0 +1,73 @@ +/** + * Relational A/B proof + no-regression gate (PGLite, default CI). + * + * Seeds the lexically-unrecoverable relational corpus, then runs the gold + * question set twice through bare hybridSearch — relational arm OFF vs ON — + * and asserts: + * 1. recall@10 on the graph-relationship family jumps materially with the + * arm on (the answers are unreachable by keyword/vector). + * 2. a non-relational content query returns the SAME results with the arm + * on vs off (the arm is a true no-op off-target — the regression gate). + * + * This is the headline proof artifact for the feature: relational queries + * that the corpus can't surface lexically jump from ~0 to high recall. + */ + +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { PGLiteEngine } from '../src/core/pglite-engine.ts'; +import { hybridSearch } from '../src/core/search/hybrid.ts'; +import { runRetrievalQuality, parseQuestionsJsonl, type SearchFn } from '../src/eval/retrieval-quality/harness.ts'; +import { seedRelationalCorpus, RELATIONAL_QUESTIONS } from './fixtures/retrieval-quality/relational/corpus.ts'; +import { readFileSync } from 'fs'; +import { join } from 'path'; + +let eng: PGLiteEngine; + +beforeAll(async () => { + eng = new PGLiteEngine(); + await eng.connect({}); + await eng.initSchema(); + await seedRelationalCorpus(eng); +}, 60_000); + +afterAll(async () => { await eng.disconnect(); }); + +const searchFnWith = (relationalRetrieval: boolean): SearchFn => async (q) => { + const results = await hybridSearch(eng, q, { limit: 10, relationalRetrieval, expansion: false }); + return results.map(r => r.slug); +}; + +describe('relational A/B', () => { + test('corpus has a meaningful question set', () => { + expect(RELATIONAL_QUESTIONS.length).toBeGreaterThanOrEqual(30); + }); + + test('relational.jsonl matches the corpus module (no drift)', () => { + const path = join(import.meta.dir, 'fixtures/retrieval-quality/relational/relational.jsonl'); + const fromFile = parseQuestionsJsonl(readFileSync(path, 'utf8')); + expect(JSON.stringify(fromFile)).toBe(JSON.stringify(RELATIONAL_QUESTIONS)); + }); + + test('recall@10 lifts materially with the relational arm ON', async () => { + const off = await runRetrievalQuality(RELATIONAL_QUESTIONS, searchFnWith(false)); + const on = await runRetrievalQuality(RELATIONAL_QUESTIONS, searchFnWith(true)); + + const offFam = off.families.find(f => f.family === 'graph-relationship')!; + const onFam = on.families.find(f => f.family === 'graph-relationship')!; + + // Answers are lexically unrecoverable → baseline recall is near zero. + expect(offFam.recall_at_10).toBeLessThan(0.25); + // Typed-edge arm surfaces them → large lift. + expect(onFam.recall_at_10).toBeGreaterThan(0.75); + expect(onFam.recall_at_10 - offFam.recall_at_10).toBeGreaterThan(0.5); + // Hit@3 should also rise sharply. + expect(onFam.hit_at_3).toBeGreaterThan(offFam.hit_at_3); + }, 120_000); + + test('no-regression: non-relational query is identical arm-on vs arm-off', async () => { + const q = 'early-stage venture fund first checks'; + const off = await searchFnWith(false)(q); + const on = await searchFnWith(true)(q); + expect(on).toEqual(off); + }, 60_000); +});