feat(search): relational recall arm + federation key hardening

Wires edge-derived candidates into bare hybridSearch as a FOURTH RRF arm
(relational-recall.ts): parse the original query -> scope-aware,
confidence-gated seed resolution (drops fallback_slugify; never traverses
from a guess) -> relationalFanout -> batch-hydrate, reinforcing each page's
REAL canonical chunk (page-level key for chunkless entity pages) ->
--explain attribution + fail-open audit row. Text-only (no-op in image
mode); pure no-op for non-relational queries; rides every downstream stage
(cosine, post-fusion boosts, dedup, reranker, autocut, token budget).

Mode wiring: relationalRetrieval + relational_retrieval_depth knobs
(conservative off; balanced/tokenmax on; depth 2), per-call thread-through
in both bare + cached paths, KNOBS_HASH_VERSION 9->10 (rel=/reld=), config
keys, modes-dashboard descriptions, and a `relational` param on the query op.

Federation hardening (structural, engine-wide): the RRF/dedup key now
carries source_id via a shared rrfKey() (fixes a latent cross-source
collapse where same-slug pages in different sources merged), and the query
cache scopes by a canonical source-set key (cacheScopeKey) so a federated
read can't be served a single-source row.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-06-07 18:08:51 -07:00
co-authored by Claude Opus 4.8
parent 4dfcd75599
commit 78d2d3dc18
9 changed files with 610 additions and 8 deletions
+3
View File
@@ -71,6 +71,9 @@ const KNOB_DESCRIPTIONS: Record<keyof ModeBundle, string> = {
// v0.42.3.0 autocut
autocut: 'Score-discontinuity result-sizing (cuts at the rerank-score cliff; no-op without a reranker)',
autocut_jump: 'Autocut sensitivity: min normalized score gap that counts as a cliff (0..1, 0.20 default)',
// v0.43 relational recall
relationalRetrieval: 'Typed-edge relational recall arm (relational queries walk the graph; no-op otherwise)',
relational_retrieval_depth: 'Max hops for relational traversal (1..3, 2 default)',
};
interface SearchModesReport {
+7
View File
@@ -1425,6 +1425,11 @@ const query: Operation = {
" TRUE is redundant in default mode (it's already on); it only matters to override a brain whose config turned autocut off.\n" +
"Safe by construction: never returns empty when there are matches, only applies to the first page (omit when paginating), and is a no-op when no reranker scored the results (so it can't cut on an untrustworthy signal). Distinct from `adaptive_return`: autocut cuts on the score cliff; adaptive_return caps by question intent. Leave both unset for the smart default.",
},
relational: {
type: 'boolean',
description:
"v0.43 — relational recall arm. SMART DEFAULT (on in balanced/tokenmax). When the question is about a RELATIONSHIP ('who invested in widget-co', 'who introduced me to alice', 'what connects fund-a and fund-b'), the brain resolves the named entity and walks its typed-edge graph (invested_in, works_at, founded, …), surfacing the answer even when no passage mentions both sides. Pure no-op for non-relational questions. Pass FALSE to force lexical/vector-only retrieval (e.g. debugging why a graph answer appeared). You almost never set this.",
},
},
handler: async (ctx, p) => {
const startedAt = Date.now();
@@ -1522,6 +1527,8 @@ const query: Operation = {
// v0.42.3.0 — autocut ceiling override. Omitted = smart default (ON in
// reranked modes). `false` forces the full top-K.
autocut: typeof p.autocut === 'boolean' ? (p.autocut as boolean) : undefined,
// v0.43 — relational recall override. Omitted = smart default (mode bundle).
relationalRetrieval: typeof p.relational === 'boolean' ? (p.relational as boolean) : undefined,
});
const latency_ms = Date.now() - startedAt;
+76 -4
View File
@@ -23,6 +23,7 @@ import {
type AdaptiveReturnDecision,
} from './return-policy.ts';
import { applyAutocut, type AutocutDecision } from './autocut.ts';
import { buildRelationalArm } from './relational-recall.ts';
import { loadConfigWithEngine } from '../config.ts';
import { dedupResults } from './dedup.ts';
import { applyReranker } from './rerank.ts';
@@ -664,6 +665,9 @@ export async function applyAliasHop(
export interface HybridSearchOpts extends SearchOpts {
expansion?: boolean;
/** v0.43 — observability sink for the relational recall arm (fired/no-op,
* kind, seeds resolved, candidates, errored). Best-effort. */
onRelationalMeta?: (meta: import('./relational-recall.ts').RelationalArmMeta) => void;
/**
* T4/D5 — per-call search-mode selector (one of SEARCH_MODES). Selects the
* whole mode bundle for this call, overriding the server-configured mode.
@@ -806,6 +810,11 @@ export async function hybridSearch(
// Non-boolean AutocutInput shapes (Partial) aren't a v1 per-call surface,
// so only the boolean toggle threads here.
autocut: typeof opts?.autocut === 'boolean' ? opts.autocut : undefined,
// v0.43 — relational recall per-call thread-through. Per-call wins over
// config override wins over mode bundle; without this the A/B eval gate
// would be a no-op (both branches resolve to the same mode default).
relationalRetrieval: opts?.relationalRetrieval,
relational_retrieval_depth: opts?.relationalRetrievalDepth,
},
});
@@ -1266,6 +1275,28 @@ export async function hybridSearch(
...vectorLists.map(list => ({ list, k: vectorK })),
{ list: keywordResults, k: keywordK },
];
// v0.43 — relational recall arm (fourth RRF arm). Fires only when the mode
// enables it AND the query is text-shaped (E4: no-op in image-only mode).
// Parsed from the ORIGINAL query (never expanded variants) so it's
// deterministic. Empty list = pure no-op for non-relational queries. Rides
// every downstream stage (cosine re-score, post-fusion boosts, dedup,
// reranker, autocut, token budget) like any other arm.
if (resolvedMode.relationalRetrieval && effectiveModality !== 'image') {
const relationalList = await buildRelationalArm(engine, query, {
sourceId: opts?.sourceId,
sourceIds: opts?.sourceIds,
depth: resolvedMode.relational_retrieval_depth,
limit: opts?.limit ?? resolvedMode.searchLimit,
onMeta: opts?.onRelationalMeta,
});
if (relationalList.length > 0) {
// Neutral weight: competes evenly with keyword/vector (baseRrfK), not
// dominating. Calibrated against the Commit 5 A/B.
allLists.push({ list: relationalList, k: baseRrfK });
}
}
let fused = rrfFusionWeighted(allLists, detail !== 'high');
// Cosine re-scoring before dedup so semantically better chunks survive.
@@ -1511,6 +1542,11 @@ export async function hybridSearchCached(
// this, an `autocut:false` (full top-K) call could be served a trimmed
// autocut-on cache row, or vice versa.
autocut: typeof opts?.autocut === 'boolean' ? opts.autocut : undefined,
// v0.43 — relational recall per-call thread-through. Per-call wins over
// config override wins over mode bundle; without this the A/B eval gate
// would be a no-op (both branches resolve to the same mode default).
relationalRetrieval: opts?.relationalRetrieval,
relational_retrieval_depth: opts?.relationalRetrievalDepth,
},
});
// v0.36 (D8 / CDX-2 + codex /ship #4): resolve column for the cache
@@ -1607,7 +1643,7 @@ export async function hybridSearchCached(
}
if (!skipCache && queryEmbedding && cacheStatus !== 'disabled') {
const hit = await cache.lookup(queryEmbedding, { sourceId: opts?.sourceId, knobsHash: cacheKnobsHash });
const hit = await cache.lookup(queryEmbedding, { sourceId: cacheScopeKey(opts), knobsHash: cacheKnobsHash });
if (hit.hit && hit.results) {
cacheStatus = 'hit';
cacheSimilarity = hit.similarity;
@@ -1708,7 +1744,7 @@ export async function hybridSearchCached(
) {
trackCacheWrite(
cache
.store(query, queryEmbedding, results, finalMeta, { sourceId: opts?.sourceId, knobsHash: cacheKnobsHash })
.store(query, queryEmbedding, results, finalMeta, { sourceId: cacheScopeKey(opts), knobsHash: cacheKnobsHash })
.catch(() => { /* swallow */ }),
);
}
@@ -1716,6 +1752,42 @@ export async function hybridSearchCached(
return budgeted;
}
/**
* RRF/dedup identity for a result row, at chunk granularity.
*
* Includes `source_id` so two same-slug pages in different federated sources
* don't collapse into one fusion entry (the same composite-key discipline
* `dedup.ts:pageKey` already uses at page granularity). Pre-fix the key was
* `slug:chunk_id`, which silently merged cross-source rows and let a
* synthetic chunkless row (chunk_id null) key on a text prefix; the
* `(source_id, slug, chunk_id)` shape is collision-safe for both.
*/
function rrfKey(r: SearchResult): string {
const source = r.source_id ?? 'default';
return `${source}:${r.slug}:${r.chunk_id ?? r.chunk_text.slice(0, 50)}`;
}
/**
* Canonical query-cache scope key.
*
* The semantic cache stores results keyed by `(scope, query, knobs_hash)`.
* A federated search (`sourceIds`) reads a different graph than a
* single-source one, so the two must never share a cache row. Pre-fix the
* cache only saw scalar `sourceId`; a federated query fell through to
* `'default'` and could cross-serve an unrelated scope.
*
* - federated (sourceIds set) → `__set__:` + sorted, comma-joined ids
* (order-independent; two different source-sets get distinct keys)
* - scalar sourceId → the id itself (single-source unchanged)
* - unscoped → `'default'` (single-source brains unchanged)
*/
export function cacheScopeKey(opts?: { sourceId?: string; sourceIds?: string[] }): string {
if (opts?.sourceIds && opts.sourceIds.length > 0) {
return '__set__:' + [...opts.sourceIds].sort().join(',');
}
return opts?.sourceId ?? 'default';
}
/**
* v0.32.x search-lite — weighted RRF. Each list contributes with its own
* effective k value, which lets intent weighting bias keyword vs vector
@@ -1731,7 +1803,7 @@ export function rrfFusionWeighted(
for (const { list, k } of lists) {
for (let rank = 0; rank < list.length; rank++) {
const r = list[rank];
const key = `${r.slug}:${r.chunk_id ?? r.chunk_text.slice(0, 50)}`;
const key = rrfKey(r);
const existing = scores.get(key);
const rrfScore = 1 / (k + rank);
@@ -1771,7 +1843,7 @@ export function rrfFusion(lists: SearchResult[][], k: number, applyBoost = true)
for (const list of lists) {
for (let rank = 0; rank < list.length; rank++) {
const r = list[rank];
const key = `${r.slug}:${r.chunk_id ?? r.chunk_text.slice(0, 50)}`;
const key = rrfKey(r);
const existing = scores.get(key);
const rrfScore = 1 / (k + rank);
+55 -1
View File
@@ -261,6 +261,18 @@ export interface ModeBundle {
* run. Override: `search.autocut_jump` config → mode bundle.
*/
autocut_jump: number;
/**
* v0.43 — relational recall arm. When on, a relational query ("who invested
* in widget-co", "what connects fund-a and fund-b") resolves its seed
* entity and walks the typed-edge graph, injecting edge-derived candidates
* as a fourth RRF arm. Pure no-op for non-relational queries. Default OFF
* for conservative; ON for balanced/tokenmax. Override path: per-call
* SearchOpts.relationalRetrieval → `search.relational_retrieval` config →
* mode bundle. See src/core/search/relational-recall.ts.
*/
relationalRetrieval: boolean;
/** v0.43 — max hops for relational traversal. Default 2, hard-capped at 3. */
relational_retrieval_depth: number;
}
/**
@@ -309,6 +321,10 @@ export const MODE_BUNDLES: Readonly<Record<SearchMode, Readonly<ModeBundle>>> =
// v0.42.3.0 — autocut OFF: conservative has no reranker, so no trustworthy
// cliff signal exists (autocut would no-op). Explicit for clarity.
autocut: false,
// v0.43 — relational recall OFF for conservative (cost-sensitive tier,
// matches graph_signals posture). Power users opt in per-call.
relationalRetrieval: false,
relational_retrieval_depth: 2,
autocut_jump: 0.2,
}),
balanced: Object.freeze({
@@ -363,6 +379,10 @@ export const MODE_BUNDLES: Readonly<Record<SearchMode, Readonly<ModeBundle>>> =
contextual_retrieval_disabled: false,
// v0.42.3.0 — autocut ON (reranker fires; cliff signal is trustworthy).
autocut: true,
// v0.43 — relational recall ON (contingent on the no-regression gate;
// ships default-false everywhere if the gate flags any regression).
relationalRetrieval: true,
relational_retrieval_depth: 2,
autocut_jump: 0.2,
}),
tokenmax: Object.freeze({
@@ -411,6 +431,9 @@ export const MODE_BUNDLES: Readonly<Record<SearchMode, Readonly<ModeBundle>>> =
contextual_retrieval_disabled: false,
// v0.42.3.0 — autocut ON.
autocut: true,
// v0.43 — relational recall ON for tokenmax (max-recall tier).
relationalRetrieval: true,
relational_retrieval_depth: 2,
autocut_jump: 0.2,
}),
});
@@ -462,6 +485,9 @@ export interface SearchKeyOverrides {
contextual_retrieval_disabled?: boolean;
// v0.42.3.0 — autocut overrides.
autocut?: boolean;
// v0.43 — relational recall overrides.
relationalRetrieval?: boolean;
relational_retrieval_depth?: number;
autocut_jump?: number;
}
@@ -509,6 +535,9 @@ export interface SearchPerCallOpts {
// numeric per-call knob threaded through the bundle.
autocut?: boolean;
autocut_jump?: number;
// v0.43 — relational recall per-call overrides.
relationalRetrieval?: boolean;
relational_retrieval_depth?: number;
}
/**
@@ -601,6 +630,9 @@ export function resolveSearchMode(input: ResolveSearchModeInput): ResolvedSearch
// v0.42.3.0 — autocut resolved via the same pick chain.
autocut: pick('autocut'),
autocut_jump: pick('autocut_jump'),
// v0.43 — relational recall resolved via the same pick chain.
relationalRetrieval: pick('relationalRetrieval'),
relational_retrieval_depth: pick('relational_retrieval_depth'),
resolved_mode,
mode_valid: valid,
};
@@ -706,7 +738,7 @@ export function attributeKnob<K extends keyof ModeBundle>(
// to take effect immediately (one-time global cache cold-miss on upgrade; refills
// within cache.ttl_seconds). Same cache-key-contamination convention as the
// autocut / title_boost / graph_signals bumps above.
export const KNOBS_HASH_VERSION = 9;
export const KNOBS_HASH_VERSION = 10;
/**
* v0.36 (D8 / CDX-2) — second-arg context for the cache key. The
@@ -814,6 +846,14 @@ export function knobsHash(
// etc.) so a partial-knobs caller (tests passing a minimal literal) can't
// crash the hash. Typed callers always carry the field.
`acj=${(knobs.autocut_jump ?? 0.2).toFixed(2)}`,
// v=10 additions (v0.43, append-only): relational recall arm. A
// relational-on write (edge-seeded result set) must NOT be served to a
// relational-off lookup — same contamination class as graph_signals. The
// depth changes the candidate set too, so it folds in as well. ONE-TIME
// cold-miss on upgrade as v=9 rows become unreachable; pinned by
// test/model-pricing.test.ts-style drift guards and the mode tests.
`rel=${knobs.relationalRetrieval ? 1 : 0}`,
`reld=${knobs.relational_retrieval_depth ?? 2}`,
];
const h = createHash('sha256');
h.update(parts.join('|'));
@@ -981,6 +1021,17 @@ export function loadOverridesFromConfig(
if (Number.isFinite(n) && n > 0 && n <= 1) out.autocut_jump = n;
}
// v0.43 — relational recall arm.
const rel = get('search.relational_retrieval');
if (rel !== undefined) {
out.relationalRetrieval = rel === '1' || rel.toLowerCase() === 'true';
}
const reld = get('search.relational_retrieval_depth');
if (reld !== undefined) {
const n = parseInt(reld, 10);
if (Number.isFinite(n) && n >= 1 && n <= 3) out.relational_retrieval_depth = n;
}
return out;
}
@@ -1019,6 +1070,9 @@ export const SEARCH_MODE_CONFIG_KEYS: ReadonlyArray<string> = Object.freeze([
'search.contextual_retrieval_disabled',
// v0.42.3.0 autocut
'search.autocut',
// v0.43 relational recall
'search.relational_retrieval',
'search.relational_retrieval_depth',
'search.autocut_jump',
]);
+232
View File
@@ -0,0 +1,232 @@
/**
* Relational recall arm (typed-edge retrieval, v0.43).
*
* Turns a relational query into a ranked list of edge-derived candidates that
* hybridSearch injects as a FOURTH RRF arm (alongside keyword + vector), so a
* relationship answer competes for ranking instead of relying on lexical or
* vector similarity to surface it.
*
* Flow: parseRelationalQuery → resolve seed entity(ies) (scope-aware,
* confidence-gated) → engine.relationalFanout (within-source, deterministic)
* → batch-hydrate to SearchResult rows (reinforcing each page's REAL canonical
* chunk; page-level key for chunkless entity pages).
*
* Determinism: parses the ORIGINAL query (never an LLM-expanded variant);
* traversal + resolution are deterministic. Fail-open: any error returns an
* empty arm + an audit row, never breaking the search hot path.
*
* Federation (E2=A): resolves the seed in every in-scope source and fans out
* from each; traversal stays WITHIN each source (no cross-boundary edges in
* v1). Confidence gate (D3): a seed that only `fallback_slugify`-resolves is
* dropped, so the arm never traverses from an invented slug. The tier-2
* resolution-margin gate is a filed TODO.
*
* Tested in test/relational-recall.test.ts.
*/
import type { BrainEngine } from '../engine.ts';
import type { SearchResult, PageType, RelationalFanoutRow } from '../types.ts';
import { createAuditWriter } from '../audit/audit-writer.ts';
import { resolveEntitySlugWithSource } from '../entities/resolve.ts';
import { parseRelationalQuery, type RelationalQuery, type RelationVocab } from './relational-intent.ts';
export interface RelationalArmOpts {
sourceId?: string;
sourceIds?: string[];
depth?: number;
limit?: number;
vocab?: RelationVocab;
onMeta?: (meta: RelationalArmMeta) => void;
}
export interface RelationalArmMeta {
fired: boolean;
kind: RelationalQuery['kind'] | null;
seeds_resolved: number;
candidates: number;
errored: boolean;
duration_ms: number;
}
interface RelationalFailureEvent {
ts: string;
error_summary: string;
query_kind: string;
}
const failureWriter = createAuditWriter<RelationalFailureEvent>({
featureName: 'relational-recall-failures',
errorLabel: 'gbrain',
errorTrailer: '; search continues',
});
/** Recent relational-arm fail-open events — consumed by doctor + search stats. */
export function readRecentRelationalFailures(days = 7, now: Date = new Date()): RelationalFailureEvent[] {
return failureWriter.readRecent(days, now);
}
function truncate(msg: string, max = 200): string {
return msg.length <= max ? msg : msg.slice(0, max - 1) + '…';
}
/** Sources to resolve a seed against. Federated → the set; scalar → [id];
* unscoped/__all__ → ['default'] (single-source brains; multi-source
* enumeration under __all__ is a v1 limitation). */
function scopeSources(opts: RelationalArmOpts): string[] {
if (opts.sourceIds && opts.sourceIds.length > 0) return opts.sourceIds;
if (opts.sourceId && opts.sourceId !== '__all__') return [opts.sourceId];
return ['default'];
}
/** Resolve a seed phrase to all in-scope (source_id, slug) pairs that
* resolve to a REAL page (confidence gate D3 tier-1: drop fallback_slugify). */
async function resolveSeedScoped(
engine: BrainEngine,
sources: string[],
phrase: string,
): Promise<Array<{ source_id: string; slug: string }>> {
const out: Array<{ source_id: string; slug: string }> = [];
const seen = new Set<string>();
for (const sid of sources) {
const r = await resolveEntitySlugWithSource(engine, sid, phrase);
if (!r || r.source === 'fallback_slugify') continue;
const key = `${sid}:${r.slug}`;
if (seen.has(key)) continue;
seen.add(key);
out.push({ source_id: sid, slug: r.slug });
}
return out;
}
/** Batch-hydrate fanout rows into SearchResult rows in fanout (ranked) order. */
async function hydrate(
engine: BrainEngine,
rows: RelationalFanoutRow[],
seedSlug: string,
): Promise<SearchResult[]> {
if (rows.length === 0) return [];
const slugs = Array.from(new Set(rows.map(r => r.slug)));
const pageRows = await engine.executeRaw<{
page_id: number; slug: string; source_id: string; title: string; type: string; synopsis: string | null;
}>(
`SELECT p.id AS page_id, p.slug, p.source_id, p.title, p.type,
LEFT(p.compiled_truth, 240) AS synopsis
FROM pages p
WHERE p.slug = ANY($1::text[]) AND p.deleted_at IS NULL`,
[slugs],
);
const byKey = new Map<string, typeof pageRows[number]>();
for (const pr of pageRows) byKey.set(`${pr.source_id}:${pr.slug}`, pr);
const out: SearchResult[] = [];
for (const r of rows) {
const pr = byKey.get(`${r.source_id}:${r.slug}`);
if (!pr) continue;
out.push({
slug: r.slug,
page_id: pr.page_id,
title: pr.title,
type: pr.type as PageType,
chunk_text: pr.synopsis ?? r.slug,
chunk_source: 'compiled_truth',
// E1: reinforce the page's REAL canonical chunk; F3: chunkless entity
// pages key page-level (chunk_id 0 → rrfKey `source:slug:0`, stable and
// collision-safe; SERIAL chunk ids start at 1 so 0 never aliases a real chunk).
chunk_id: r.canonical_chunk_id ?? 0,
chunk_index: 0,
score: 0, // rank-based: RRF derives score from list position
stale: false,
source_id: r.source_id,
relational_via_link_types: r.via_link_types,
relational_seed: seedSlug,
relational_hop: r.hop,
relational_path: r.path,
});
}
return out;
}
/**
* Build the relational recall arm. Returns an empty list (pure no-op) when the
* query isn't relational or no seed resolves. Never throws.
*/
export async function buildRelationalArm(
engine: BrainEngine,
query: string,
opts: RelationalArmOpts = {},
): Promise<SearchResult[]> {
const startedAt = Date.now();
const meta: RelationalArmMeta = {
fired: false, kind: null, seeds_resolved: 0, candidates: 0, errored: false, duration_ms: 0,
};
const finish = (list: SearchResult[]) => {
meta.candidates = list.length;
meta.duration_ms = Date.now() - startedAt;
opts.onMeta?.(meta);
return list;
};
const parsed = parseRelationalQuery(query, opts.vocab);
if (!parsed) return finish([]);
meta.kind = parsed.kind;
try {
const sources = scopeSources(opts);
const fanoutOpts = {
linkTypes: parsed.linkTypes,
direction: parsed.direction,
depth: opts.depth,
limit: opts.limit,
};
if (parsed.kind === 'connects' && parsed.seeds.length === 2) {
// Resolve both endpoints; both must resolve or the arm no-ops.
const resA = await resolveSeedScoped(engine, sources, parsed.seeds[0]);
const resB = await resolveSeedScoped(engine, sources, parsed.seeds[1]);
if (resA.length === 0 || resB.length === 0) return finish([]);
meta.seeds_resolved = resA.length + resB.length;
const perSource = (rs: typeof resA) => ({
sourceId: rs.length === 1 ? rs[0].source_id : undefined,
sourceIds: rs.length > 1 ? Array.from(new Set(rs.map(x => x.source_id))) : undefined,
slugs: Array.from(new Set(rs.map(x => x.slug))),
});
const a = perSource(resA);
const b = perSource(resB);
const fanA = await engine.relationalFanout(a.slugs, { ...fanoutOpts, sourceId: a.sourceId, sourceIds: a.sourceIds });
const fanB = await engine.relationalFanout(b.slugs, { ...fanoutOpts, sourceId: b.sourceId, sourceIds: b.sourceIds });
// Shared midpoints: nodes reachable from BOTH endpoints (exclude the
// endpoints themselves). Ordered by combined hop.
const bByKey = new Map(fanB.map(r => [`${r.source_id}:${r.slug}`, r] as const));
const endpointSlugs = new Set([...a.slugs, ...b.slugs]);
const shared = fanA
.filter(r => bByKey.has(`${r.source_id}:${r.slug}`) && !endpointSlugs.has(r.slug))
.map(r => ({ row: r, combined: r.hop + bByKey.get(`${r.source_id}:${r.slug}`)!.hop }))
.sort((x, y) => x.combined - y.combined || x.row.slug.localeCompare(y.row.slug))
.map(x => x.row);
const list = await hydrate(engine, shared, parsed.seeds.join(' ↔ '));
meta.fired = list.length > 0;
return finish(list);
}
// who_rel / who_at / intro: single logical seed (may resolve in N sources).
const resolved = await resolveSeedScoped(engine, sources, parsed.seeds[0]);
if (resolved.length === 0) return finish([]);
meta.seeds_resolved = resolved.length;
const slugs = Array.from(new Set(resolved.map(r => r.slug)));
const srcIds = Array.from(new Set(resolved.map(r => r.source_id)));
const rows = await engine.relationalFanout(slugs, {
...fanoutOpts,
sourceId: srcIds.length === 1 ? srcIds[0] : undefined,
sourceIds: srcIds.length > 1 ? srcIds : undefined,
});
const list = await hydrate(engine, rows, resolved[0].slug);
meta.fired = list.length > 0;
return finish(list);
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
failureWriter.log({ error_summary: truncate(msg), query_kind: parsed.kind });
meta.errored = true;
return finish([]);
}
}
+42
View File
@@ -0,0 +1,42 @@
/**
* Query-cache scope key (federation hardening).
*
* A federated search reads a different graph than a single-source one, so
* the semantic cache must key them apart. `cacheScopeKey` produces an
* order-independent key for federated scopes and leaves single-source
* brains on their existing key (scalar id or 'default'), so single-source
* cache hit-rate is unchanged.
*/
import { describe, test, expect } from 'bun:test';
import { cacheScopeKey } from '../src/core/search/hybrid.ts';
describe('cacheScopeKey', () => {
test('unscoped → default (single-source unchanged)', () => {
expect(cacheScopeKey(undefined)).toBe('default');
expect(cacheScopeKey({})).toBe('default');
});
test('scalar sourceId → itself (single-source unchanged)', () => {
expect(cacheScopeKey({ sourceId: 'host' })).toBe('host');
});
test('federated sourceIds → order-independent set key', () => {
const k1 = cacheScopeKey({ sourceIds: ['team-b', 'team-a', 'host'] });
const k2 = cacheScopeKey({ sourceIds: ['host', 'team-a', 'team-b'] });
expect(k1).toBe(k2); // order does not matter
expect(k1).toBe('__set__:host,team-a,team-b');
});
test('different source-sets do NOT share a key', () => {
const a = cacheScopeKey({ sourceIds: ['host', 'team-a'] });
const b = cacheScopeKey({ sourceIds: ['host', 'team-b'] });
expect(a).not.toBe(b);
});
test('federated set key is distinct from any single scalar key', () => {
const set = cacheScopeKey({ sourceIds: ['host'] });
const scalar = cacheScopeKey({ sourceId: 'host' });
expect(set).not.toBe(scalar); // a 1-element set still cannot serve a scalar read
});
});
+70
View File
@@ -0,0 +1,70 @@
/**
* Relational recall arm integration tests (PGLite, default CI).
*
* End-to-end through buildRelationalArm: parse → resolve seed → fanout →
* hydrate. Pins the lexically-unrecoverable win (the investor page never names
* the company; only the invested_in edge connects them), the non-relational
* no-op, attribution stamping, and fail-open.
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { buildRelationalArm } from '../src/core/search/relational-recall.ts';
import type { ChunkInput } from '../src/core/types.ts';
let eng: PGLiteEngine;
beforeAll(async () => {
eng = new PGLiteEngine();
await eng.connect({});
await eng.initSchema();
await eng.putPage('companies/widget-co', { type: 'company', title: 'Widget Co', compiled_truth: 'A payments company.', timeline: '' });
// The investor's body deliberately NEVER mentions Widget Co — only the edge connects them.
await eng.putPage('people/alice-example', { type: 'person', title: 'Alice Example', compiled_truth: 'Alice is a seed-stage investor based in Lisbon.', timeline: '' });
await eng.upsertChunks('people/alice-example', [{
chunk_index: 0, chunk_text: 'Alice is a seed-stage investor based in Lisbon.',
chunk_source: 'compiled_truth', embedding: new Float32Array(1536), token_count: 8,
}] satisfies ChunkInput[]);
await eng.addLink('people/alice-example', 'companies/widget-co', '', 'invested_in', 'manual');
}, 60_000);
afterAll(async () => { await eng.disconnect(); });
describe('buildRelationalArm', () => {
test('surfaces the edge answer that lexical search would miss', async () => {
const list = await buildRelationalArm(eng, 'who invested in widget-co');
const alice = list.find(r => r.slug === 'people/alice-example');
expect(alice).toBeDefined();
expect(alice!.relational_via_link_types).toEqual(['invested_in']);
expect(alice!.relational_hop).toBe(1);
expect(alice!.relational_seed).toBe('companies/widget-co');
// chunk-backed page → reinforces a REAL chunk id (not synthetic 0).
expect(alice!.chunk_id).toBeGreaterThan(0);
});
test('non-relational query is a pure no-op', async () => {
const meta: { fired?: boolean } = {};
const list = await buildRelationalArm(eng, 'summary of the payments roadmap', { onMeta: m => { meta.fired = m.fired; } });
expect(list).toEqual([]);
expect(meta.fired).toBe(false);
});
test('unresolvable seed → no-op (never traverse from a guess)', async () => {
const list = await buildRelationalArm(eng, 'who invested in nonexistent-phantom-xyz');
expect(list).toEqual([]);
});
test('fail-open: fanout error returns [] + errored meta, never throws', async () => {
const original = eng.relationalFanout.bind(eng);
let captured: { errored?: boolean } = {};
eng.relationalFanout = async () => { throw new Error('boom'); };
try {
const list = await buildRelationalArm(eng, 'who invested in widget-co', { onMeta: m => { captured = m; } });
expect(list).toEqual([]);
expect(captured.errored).toBe(true);
} finally {
eng.relationalFanout = original;
}
});
});
+70
View File
@@ -0,0 +1,70 @@
/**
* RRF fusion key is source-aware (federation hardening).
*
* Pre-fix the RRF/dedup key was `slug:chunk_id`, which collapsed two
* same-slug pages living in different federated sources into one fusion
* entry — a cross-source recall bug. The key is now
* `(source_id, slug, chunk_id)`, matching `dedup.ts:pageKey`'s composite
* discipline at chunk granularity. These tests pin that behavior for both
* `rrfFusion` and `rrfFusionWeighted` and confirm single-source ranking is
* unchanged.
*/
import { describe, test, expect } from 'bun:test';
import { rrfFusion, rrfFusionWeighted } from '../src/core/search/hybrid.ts';
import type { SearchResult } from '../src/core/types.ts';
function makeResult(overrides: Partial<SearchResult> = {}): SearchResult {
return {
slug: 'test-page',
page_id: 1,
title: 'Test',
type: 'concept',
chunk_text: 'unique chunk text',
chunk_source: 'timeline', // avoid compiled_truth 2x boost noise
chunk_id: 1,
chunk_index: 0,
score: 0.5,
stale: false,
...overrides,
};
}
describe('RRF key is source-aware', () => {
test('same slug + chunk_id in DIFFERENT sources do NOT collapse', () => {
const a = makeResult({ slug: 'people/alice', chunk_id: 1, source_id: 'team-a', chunk_text: 'x' });
const b = makeResult({ slug: 'people/alice', chunk_id: 1, source_id: 'team-b', chunk_text: 'y' });
const out = rrfFusion([[a, b]], 60);
expect(out.length).toBe(2);
expect(new Set(out.map(r => r.source_id))).toEqual(new Set(['team-a', 'team-b']));
});
test('same slug + chunk_id + SAME source DO collapse (reinforce across lists)', () => {
const a = makeResult({ slug: 'people/alice', chunk_id: 1, source_id: 'team-a', chunk_text: 'x' });
const b = makeResult({ slug: 'people/alice', chunk_id: 1, source_id: 'team-a', chunk_text: 'x' });
const out = rrfFusion([[a], [b]], 60);
expect(out.length).toBe(1);
});
test('single-source (no source_id → "default") ranking unchanged', () => {
const a = makeResult({ slug: 'a', chunk_id: 1, chunk_text: 'aaa' });
const b = makeResult({ slug: 'b', chunk_id: 2, chunk_text: 'bbb' });
const out = rrfFusion([[a, b]], 60);
expect(out.length).toBe(2);
expect(out[0].slug).toBe('a'); // rank-0 wins, as before the key change
});
test('chunkless rows (null chunk_id) stay distinct per source via text prefix', () => {
const a = makeResult({ slug: 'people/alice', chunk_id: undefined as unknown as number, source_id: 'team-a', chunk_text: 'alice page' });
const b = makeResult({ slug: 'people/alice', chunk_id: undefined as unknown as number, source_id: 'team-b', chunk_text: 'alice page' });
const out = rrfFusion([[a, b]], 60);
expect(out.length).toBe(2);
});
test('weighted RRF is also source-aware', () => {
const a = makeResult({ slug: 'people/alice', chunk_id: 1, source_id: 'team-a' });
const b = makeResult({ slug: 'people/alice', chunk_id: 1, source_id: 'team-b' });
const out = rrfFusionWeighted([{ list: [a, b], k: 60 }]);
expect(out.length).toBe(2);
});
});
+55 -3
View File
@@ -77,6 +77,9 @@ describe('SEARCH_MODES + MODE_BUNDLES canonical shape', () => {
// v0.42.3.0 — autocut OFF for conservative (no reranker).
autocut: false,
autocut_jump: 0.2,
// v0.43 — relational recall OFF for conservative.
relationalRetrieval: false,
relational_retrieval_depth: 2,
});
});
@@ -106,6 +109,9 @@ describe('SEARCH_MODES + MODE_BUNDLES canonical shape', () => {
// v0.42.3.0 — autocut ON.
autocut: true,
autocut_jump: 0.2,
// v0.43 — relational recall ON for balanced.
relationalRetrieval: true,
relational_retrieval_depth: 2,
});
});
@@ -133,6 +139,9 @@ describe('SEARCH_MODES + MODE_BUNDLES canonical shape', () => {
// v0.42.3.0 — autocut ON.
autocut: true,
autocut_jump: 0.2,
// v0.43 — relational recall ON for tokenmax.
relationalRetrieval: true,
relational_retrieval_depth: 2,
});
});
@@ -392,7 +401,9 @@ describe('knobsHash determinism + cross-mode separation (CDX-4)', () => {
// archive/ demote (search-exclude policy change isn't in the hash, so the
// version bump is what invalidates archive-excluded cache rows). A query
// must not be served from a cache row written before the policy change.
expect(KNOBS_HASH_VERSION).toBe(9);
// v0.43: bumped 9→10 for the relational recall arm (rel=/reld=) — a
// relational-on write must not be served to a relational-off lookup.
expect(KNOBS_HASH_VERSION).toBe(10);
});
test('T1 (codex): floor_ratio set vs unset produces DIFFERENT hashes (cache contamination prevention)', () => {
@@ -557,8 +568,8 @@ describe('v0.40.4 — graph_signals knob', () => {
});
describe('v0.42.3.0 — autocut knobs', () => {
test('KNOBS_HASH_VERSION is 9 (8→9 archive-demote, issue #1777)', () => {
expect(KNOBS_HASH_VERSION).toBe(9);
test('KNOBS_HASH_VERSION is 10 (9→10 relational recall arm, v0.43)', () => {
expect(KNOBS_HASH_VERSION).toBe(10);
});
test('bundle defaults: conservative off, balanced/tokenmax on @0.20', () => {
@@ -621,3 +632,44 @@ describe('v0.42.3.0 — autocut knobs', () => {
expect(attr.value).toBe(false);
});
});
describe('v0.43 — relational recall knobs', () => {
test('bundle defaults: conservative off, balanced/tokenmax on; depth 2', () => {
expect(MODE_BUNDLES.conservative.relationalRetrieval).toBe(false);
expect(MODE_BUNDLES.balanced.relationalRetrieval).toBe(true);
expect(MODE_BUNDLES.tokenmax.relationalRetrieval).toBe(true);
for (const m of ['conservative', 'balanced', 'tokenmax'] as const) {
expect(MODE_BUNDLES[m].relational_retrieval_depth).toBe(2);
}
});
test('per-call relationalRetrieval:false overrides bundle/config true', () => {
// bundle default true (balanced); per-call false wins
expect(resolveSearchMode({ mode: 'balanced', perCall: { relationalRetrieval: false } }).relationalRetrieval).toBe(false);
// config override true on conservative (bundle false); then per-call false beats config
expect(
resolveSearchMode({ mode: 'conservative', overrides: { relationalRetrieval: true }, perCall: { relationalRetrieval: false } }).relationalRetrieval,
).toBe(false);
// config override alone flips conservative on
expect(resolveSearchMode({ mode: 'conservative', overrides: { relationalRetrieval: true } }).relationalRetrieval).toBe(true);
});
test('loadOverridesFromConfig reads search.relational_retrieval(+_depth)', () => {
const ov = loadOverridesFromConfig({ 'search.relational_retrieval': 'true', 'search.relational_retrieval_depth': '3' });
expect(ov.relationalRetrieval).toBe(true);
expect(ov.relational_retrieval_depth).toBe(3);
// out-of-range depth is ignored (falls through to bundle)
expect(loadOverridesFromConfig({ 'search.relational_retrieval_depth': '9' }).relational_retrieval_depth).toBeUndefined();
});
test('SEARCH_MODE_CONFIG_KEYS includes the relational keys', () => {
expect(SEARCH_MODE_CONFIG_KEYS).toContain('search.relational_retrieval');
expect(SEARCH_MODE_CONFIG_KEYS).toContain('search.relational_retrieval_depth');
});
test('knobsHash: relational-on vs off differ (cache isolation)', () => {
const on = knobsHash(resolveSearchMode({ mode: 'balanced' })); // relational true
const off = knobsHash(resolveSearchMode({ mode: 'balanced', perCall: { relationalRetrieval: false } }));
expect(on).not.toBe(off);
});
});