From 8a0e956da3112fd0e8b4c009d9b3c348604b0040 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Tue, 21 Jul 2026 14:39:44 -0700 Subject: [PATCH] fix(search): weak-top floor on autocut to stop cross-source result collapse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Takeover of #1863 (rebased onto master). applyAutocut normalizes the cliff test by the top rerank score, so a weak top (~0.317 on a rare-term cross-source pool) rescales to 1.0 and a normal decay below it masquerades as a confident cliff — collapsing the pool to 1 result (`--source __all__` returned 1 while a single-source scoped query returned ~12). Fix: AutocutConfig.minTopScore (default 0.5) — below the floor autocut no-ops and preserves recall; at/above it behavior is unchanged. Threaded like autocut_jump (mode bundle → search.autocut_min_top_score config → per-call → pick-chain). Gated on rerankerEmitsNormalizedScores(effective model): raw-logit local rerankers (llama-server/Qwen3) get floor 0 since a [0,1] threshold is meaningless on an unbounded logit scale. Rebase deltas vs #1863: KNOBS_HASH_VERSION 12→13 (master took 12 for the #2825 hard-exclude fold; acmts= folds into the key as the v=13 addition) and version-pin tests/comments updated accordingly. Co-authored-by: rayers Co-Authored-By: Claude Fable 5 --- src/commands/search.ts | 1 + src/core/search/autocut.ts | 32 +++++++++ src/core/search/hybrid.ts | 17 ++++- src/core/search/mode.ts | 42 ++++++++++- src/core/search/rerank.ts | 24 +++++++ test/cross-modal-phase1.test.ts | 5 +- test/search-alias-resolved-boost.test.ts | 4 +- test/search-mode.test.ts | 42 +++++++++-- .../search/autocut-integration.serial.test.ts | 47 ++++++++++++ test/search/autocut.test.ts | 72 ++++++++++++++++++- test/search/knobs-hash-reranker.test.ts | 6 +- test/search/rerank.test.ts | 21 +++++- 12 files changed, 298 insertions(+), 15 deletions(-) diff --git a/src/commands/search.ts b/src/commands/search.ts index 971010979..8445d0a5d 100644 --- a/src/commands/search.ts +++ b/src/commands/search.ts @@ -71,6 +71,7 @@ const KNOB_DESCRIPTIONS: Record = { // v0.42.3.0 autocut autocut: 'Score-discontinuity result-sizing (cuts at the rerank-score cliff; no-op without a reranker)', autocut_jump: 'Autocut sensitivity: min normalized score gap that counts as a cliff (0..1, 0.20 default)', + autocut_min_top_score: 'Autocut weak-top floor: min top rerank score to trust the cliff (0..1, 0.50 default; 0 disables; below it autocut no-ops to preserve recall)', // v0.43 relational recall relationalRetrieval: 'Typed-edge relational recall arm (relational queries walk the graph; no-op otherwise)', relational_retrieval_depth: 'Max hops for relational traversal (1..3, 2 default)', diff --git a/src/core/search/autocut.ts b/src/core/search/autocut.ts index 2cd84fe35..e90005e30 100644 --- a/src/core/search/autocut.ts +++ b/src/core/search/autocut.ts @@ -33,6 +33,19 @@ export interface AutocutConfig { jumpRatio: number; /** Failsafe: never return fewer than this when candidates exist (≥1). */ minKeep: number; + /** + * Weak-top floor: the minimum TOP rerank score for the cliff test to be + * trusted. The cliff is measured on scores NORMALIZED by the top, so a weak + * top (e.g. 0.317 on a rare-term cross-source query) rescales to 1.0 and a + * normal decay below it masquerades as a confident cliff — collapsing a rich + * pool to 1. When the top score is below this floor, autocut no-ops (recall + * preserved); above it, the cliff is trusted as before. The reranker + * (zerank-2) is bimodal — real matches ≈0.95+, weak matches ≈0.3 — so the + * default 0.5 sits in the empty middle. 0 disables the floor (pre-fix + * behavior). Clamped to [0, 1]. Override: `search.autocut_min_top_score` + * config → mode bundle. + */ + minTopScore: number; } /** @@ -44,6 +57,8 @@ export const DEFAULT_AUTOCUT: AutocutConfig = Object.freeze({ enabled: true, jumpRatio: 0.2, minKeep: 1, + // Weak-top floor (cross-source collapse fix). See AutocutConfig.minTopScore. + minTopScore: 0.5, }); export interface AutocutDecision { @@ -79,6 +94,13 @@ export function autocutFromConfig( typeof search.autocut_min_keep === 'number' ? Math.floor(search.autocut_min_keep) : Number.NaN; if (Number.isFinite(n) && n >= 1) out.minKeep = n; } + if (search.autocut_min_top_score !== undefined) { + const n = + typeof search.autocut_min_top_score === 'number' ? search.autocut_min_top_score : Number.NaN; + // [0, 1]: 0 disables the floor, 1 pins it at the ceiling. Out-of-range + // values are IGNORED (fall through to bundle / module default). + if (Number.isFinite(n) && n >= 0 && n <= 1) out.minTopScore = n; + } return out; } @@ -160,6 +182,16 @@ export function applyAutocut( const top = Math.max(...scores); if (!Number.isFinite(top) || top <= 0) return noOp(results); + // Weak-top floor: the cliff is measured on scores normalized by `top`, so a + // weak top fabricates a confident-looking cliff (rare-term cross-source + // queries collapse 32→1). When the top isn't a confident anchor, don't trust + // the cliff — no-op and preserve recall. `?? 0.5` mirrors DEFAULT_AUTOCUT and + // the knobsHash `acmts` default so a partial-knobs literal caller computes + // the same trimmed set the cache key claims it did (omitting the field must + // not silently disable the floor). Pass minTopScore:0 to disable explicitly. + const minTopScore = cfg.minTopScore ?? 0.5; + if (top < minTopScore) return noOp(results); + // Sort a copy descending (A2: don't trust upstream order) and normalize. const sorted = [...scores].sort((a, b) => b - a); const norm = sorted.map((s) => s / top); diff --git a/src/core/search/hybrid.ts b/src/core/search/hybrid.ts index de0a95eb4..635ba548e 100644 --- a/src/core/search/hybrid.ts +++ b/src/core/search/hybrid.ts @@ -27,7 +27,7 @@ import { applyAutocut, type AutocutDecision } from './autocut.ts'; import { buildRelationalArm } from './relational-recall.ts'; import { loadConfigWithEngine } from '../config.ts'; import { dedupResults } from './dedup.ts'; -import { applyReranker } from './rerank.ts'; +import { applyReranker, rerankerEmitsNormalizedScores } from './rerank.ts'; import { autoDetectDetail, classifyQuery, isAmbiguousModalityQuery } from './query-intent.ts'; import { isTitlePhraseMatch } from './title-match.ts'; import { normalizeAlias } from './alias-normalize.ts'; @@ -1572,10 +1572,23 @@ export async function hybridSearch( // never-empty failsafe (1); jumpRatio comes from the resolved mode. let autocutDecision: AutocutDecision | undefined; if (resolvedMode.autocut && offset === 0) { + // v0.42.x — minTopScore: weak-top floor. When the top rerank score isn't a + // confident anchor, autocut no-ops (fixes the rare-term cross-source + // collapse where a weak 0.317 top fabricated a cliff and trimmed 32→1). The + // floor is a [0,1] threshold, so it only applies to rerankers that emit + // normalized scores; on raw-logit rerankers (llama-server/Qwen3) we pass 0 + // (disabled) to avoid suppressing legitimate cuts on an unbounded scale. + // Gate on the EFFECTIVE reranker model — a per-call SearchOpts.reranker + // .model override wins over the bundle default, so reading the bundle alone + // would re-open the bug for a per-call raw-logit reranker. + const effectiveRerankerModel = rerankerOpts.model ?? resolvedMode.reranker_model; + const autocutMinTopScore = rerankerEmitsNormalizedScores(effectiveRerankerModel) + ? resolvedMode.autocut_min_top_score + : 0; const r = applyAutocut( returnPool, (x) => x.rerank_score, - { enabled: true, jumpRatio: resolvedMode.autocut_jump, minKeep: 1 }, + { enabled: true, jumpRatio: resolvedMode.autocut_jump, minKeep: 1, minTopScore: autocutMinTopScore }, // Preserve alias-hop exact matches: applyAliasHop injects the canonical // page AFTER reranking, so it has no rerank_score. Without this it would // be dropped whenever autocut cuts on the scored set (Codex P1). diff --git a/src/core/search/mode.ts b/src/core/search/mode.ts index cb5a14782..a1022e41a 100644 --- a/src/core/search/mode.ts +++ b/src/core/search/mode.ts @@ -273,6 +273,16 @@ export interface ModeBundle { relationalRetrieval: boolean; /** v0.43 — max hops for relational traversal. Default 2, hard-capped at 3. */ relational_retrieval_depth: number; + /** + * v0.42.x — autocut weak-top floor: the minimum top cross-encoder rerank + * score for the cliff cut to be trusted. autocut normalizes the cliff test by + * the top score, so a weak top (e.g. 0.317 on a rare-term cross-source query) + * rescales to 1.0 and fabricates a confident cliff — collapsing a rich pool to + * 1 (the `--source __all__` collapse). Below this floor autocut no-ops (recall + * preserved). Default 0.5 (zerank-2 is bimodal: real ≈0.95+, weak ≈0.3). 0 + * disables the floor. Override: `search.autocut_min_top_score` config → bundle. + */ + autocut_min_top_score: number; } /** @@ -326,6 +336,7 @@ export const MODE_BUNDLES: Readonly>> = relationalRetrieval: false, relational_retrieval_depth: 2, autocut_jump: 0.2, + autocut_min_top_score: 0.5, }), balanced: Object.freeze({ cache_enabled: true, @@ -384,6 +395,7 @@ export const MODE_BUNDLES: Readonly>> = relationalRetrieval: true, relational_retrieval_depth: 2, autocut_jump: 0.2, + autocut_min_top_score: 0.5, }), tokenmax: Object.freeze({ cache_enabled: true, @@ -435,6 +447,7 @@ export const MODE_BUNDLES: Readonly>> = relationalRetrieval: true, relational_retrieval_depth: 2, autocut_jump: 0.2, + autocut_min_top_score: 0.5, }), }); @@ -489,6 +502,8 @@ export interface SearchKeyOverrides { relationalRetrieval?: boolean; relational_retrieval_depth?: number; autocut_jump?: number; + // v0.42.x — autocut weak-top floor override. + autocut_min_top_score?: number; } /** @@ -538,6 +553,8 @@ export interface SearchPerCallOpts { // v0.43 — relational recall per-call overrides. relationalRetrieval?: boolean; relational_retrieval_depth?: number; + // v0.42.x — autocut weak-top floor per-call override. + autocut_min_top_score?: number; } /** @@ -633,6 +650,8 @@ export function resolveSearchMode(input: ResolveSearchModeInput): ResolvedSearch // v0.43 — relational recall resolved via the same pick chain. relationalRetrieval: pick('relationalRetrieval'), relational_retrieval_depth: pick('relational_retrieval_depth'), + // v0.42.x — autocut weak-top floor resolved via the same pick chain. + autocut_min_top_score: pick('autocut_min_top_score'), resolved_mode, mode_valid: valid, }; @@ -756,7 +775,12 @@ export function attributeKnob( // slugs written by a process without it, and vice versa. Same one-time // global cold-miss pattern as the bumps above; refills within // cache.ttl_seconds (3600s default). -export const KNOBS_HASH_VERSION = 12; +// +// bump 12→13 (autocut weak-top floor): adds `acmts` (autocut_min_top_score). +// The floor changes WHETHER autocut cuts at all — a write made with one floor +// must NOT be served to a lookup at a different floor (the trimmed-vs-full set +// differs). Same one-time global cold-miss pattern; fills within cache.ttl. +export const KNOBS_HASH_VERSION = 13; /** * v0.36 (D8 / CDX-2) — second-arg context for the cache key. The @@ -888,6 +912,13 @@ export function knobsHash( // across processes. Sorted copy so ['a/','b/'] and ['b/','a/'] hash // identically; undefined falls back to 'none' for legacy callers. `hx=${ctx?.hardExcludes ? [...ctx.hardExcludes].sort().join(',') : 'none'}`, + // v=13 addition (append-only) — autocut weak-top floor shifts whether + // autocut cuts at all, so an autocut-cut write must not be served to a + // different-floor lookup. `?? 0.5` mirrors the module default for + // partial-knobs callers. 4 decimals (vs acj's 2): the floor is compared + // directly against raw rerank scores, so nearby config values (0.501 vs + // 0.504) can flip trim-vs-no-op and must not collide on the cache key. + `acmts=${(knobs.autocut_min_top_score ?? 0.5).toFixed(4)}`, ]; const h = createHash('sha256'); h.update(parts.join('|')); @@ -1054,6 +1085,13 @@ export function loadOverridesFromConfig( const n = parseFloat(acj); if (Number.isFinite(n) && n > 0 && n <= 1) out.autocut_jump = n; } + // v0.42.x — autocut weak-top floor. [0, 1]: 0 disables, 1 pins at ceiling; + // out-of-range falls through to the bundle. + const acmts = get('search.autocut_min_top_score'); + if (acmts !== undefined) { + const n = parseFloat(acmts); + if (Number.isFinite(n) && n >= 0 && n <= 1) out.autocut_min_top_score = n; + } // v0.43 — relational recall arm. const rel = get('search.relational_retrieval'); @@ -1108,6 +1146,8 @@ export const SEARCH_MODE_CONFIG_KEYS: ReadonlyArray = Object.freeze([ 'search.relational_retrieval', 'search.relational_retrieval_depth', 'search.autocut_jump', + // v0.42.x autocut weak-top floor + 'search.autocut_min_top_score', ]); /** diff --git a/src/core/search/rerank.ts b/src/core/search/rerank.ts index 863408026..08dbd1dab 100644 --- a/src/core/search/rerank.ts +++ b/src/core/search/rerank.ts @@ -39,6 +39,30 @@ export interface RerankerOpts { rerankerFn?: (input: RerankInput) => Promise; } +/** + * Does this reranker emit relevance scores normalized to roughly [0, 1]? + * + * `rerank_score` is taken verbatim from the provider response (gateway.ts — + * no sigmoid, no normalization). Cloud rerankers (ZeroEntropy zerank-*, Cohere, + * Voyage) return calibrated [0, 1] relevance. But the self-hosted + * `llama-server-reranker` path (Qwen3-Reranker GGUF via llama.cpp `--reranking`) + * returns the RAW cross-encoder logit — unbounded, frequently negative for + * non-matches and well above 1 for strong matches. + * + * This matters for autocut's weak-top floor (autocut.ts minTopScore): the floor + * is a [0, 1] threshold and is meaningless on a raw-logit scale, where it would + * either suppress legitimate cuts (a confident logit that happens to land in + * (0, floor)) or no-op entirely (logits ≫ 1). Callers gate the floor on this. + * + * Conservative: returns false ONLY for the known raw-logit local rerankers, so + * an unrecognized cloud model still gets the (correct) normalized treatment. + */ +export function rerankerEmitsNormalizedScores(model?: string): boolean { + if (!model) return true; // default reranker is ZE zerank-2 ([0,1]) + const m = model.toLowerCase(); + return !(m.includes('llama-server-reranker') || m.includes('qwen3-reranker')); +} + /** SHA-256 prefix (8 chars) of the query text for privacy-preserving audit. */ function hashQuery(query: string): string { return createHash('sha256').update(query, 'utf8').digest('hex').slice(0, 8); diff --git a/test/cross-modal-phase1.test.ts b/test/cross-modal-phase1.test.ts index 19689a324..af412365b 100644 --- a/test/cross-modal-phase1.test.ts +++ b/test/cross-modal-phase1.test.ts @@ -136,7 +136,7 @@ describe('D2 — knobsHash differs across cross-modal knob values', () => { return resolveSearchMode({ mode: 'balanced' }); } - test('KNOBS_HASH_VERSION is 12 (cross-modal still appended; 11→12 hard-exclude fold #2825)', () => { + test('KNOBS_HASH_VERSION is 13 (cross-modal still appended; 12→13 autocut weak-top floor)', () => { // v0.35 ladder: 1→2 reranker, 2→3 floor_ratio. v0.36 piggybacks on v=3 // with 7 cross-modal knobs + column/provider context. v0.40.4 (salem) + // v0.39 T21 (master) bump to v=4 for graph_signals + schema-pack fields. @@ -146,7 +146,8 @@ describe('D2 — knobsHash differs across cross-modal knob values', () => { // v0.43: 9→10 relational recall arm. #1400: 10→11 query-side input_type // finally reaches asymmetric providers — pre-fix rows were keyed on // document-side query vectors. #2825: 11→12 hard-exclude fold (hx=). - expect(KNOBS_HASH_VERSION).toBe(12); + // 12→13: autocut weak-top floor (acmts=). + expect(KNOBS_HASH_VERSION).toBe(13); }); test('flipping unified_multimodal changes the hash', () => { diff --git a/test/search-alias-resolved-boost.test.ts b/test/search-alias-resolved-boost.test.ts index 224d2d988..15cb160bd 100644 --- a/test/search-alias-resolved-boost.test.ts +++ b/test/search-alias-resolved-boost.test.ts @@ -89,7 +89,7 @@ describe('alias_resolved boost stage', () => { }); describe('KNOBS_HASH_VERSION', () => { - it('is 12 (11→12 hard-exclude fold invalidates rows written under a different exclude policy, #2825)', () => { - expect(KNOBS_HASH_VERSION).toBe(12); + it('is 13 (12→13 autocut weak-top floor acmts= added; 11→12 was #2825 hard-exclude fold)', () => { + expect(KNOBS_HASH_VERSION).toBe(13); }); }); diff --git a/test/search-mode.test.ts b/test/search-mode.test.ts index 223d112d9..0c9a42346 100644 --- a/test/search-mode.test.ts +++ b/test/search-mode.test.ts @@ -80,6 +80,7 @@ describe('SEARCH_MODES + MODE_BUNDLES canonical shape', () => { // v0.43 — relational recall OFF for conservative. relationalRetrieval: false, relational_retrieval_depth: 2, + autocut_min_top_score: 0.5, }); }); @@ -112,6 +113,7 @@ describe('SEARCH_MODES + MODE_BUNDLES canonical shape', () => { // v0.43 — relational recall ON for balanced. relationalRetrieval: true, relational_retrieval_depth: 2, + autocut_min_top_score: 0.5, }); }); @@ -142,6 +144,7 @@ describe('SEARCH_MODES + MODE_BUNDLES canonical shape', () => { // v0.43 — relational recall ON for tokenmax. relationalRetrieval: true, relational_retrieval_depth: 2, + autocut_min_top_score: 0.5, }); }); @@ -409,8 +412,9 @@ describe('knobsHash determinism + cross-mode separation (CDX-4)', () => { // must not be served to post-fix lookups. // #2825: bumped 11→12 to fold the resolved hard-exclude prefix list // (hx=) — cached rows leaked GBRAIN_SEARCH_EXCLUDE'd slugs across - // processes. - expect(KNOBS_HASH_VERSION).toBe(12); + // processes. 12→13: autocut weak-top floor (acmts=) — the floor shifts + // whether autocut cuts at all. + expect(KNOBS_HASH_VERSION).toBe(13); }); test('T1 (codex): floor_ratio set vs unset produces DIFFERENT hashes (cache contamination prevention)', () => { @@ -575,8 +579,8 @@ describe('v0.40.4 — graph_signals knob', () => { }); describe('v0.42.3.0 — autocut knobs', () => { - test('KNOBS_HASH_VERSION is 12 (11→12 hard-exclude fold, #2825)', () => { - expect(KNOBS_HASH_VERSION).toBe(12); + test('KNOBS_HASH_VERSION is 13 (12→13 autocut weak-top floor; 11→12 was #2825 hard-excludes)', () => { + expect(KNOBS_HASH_VERSION).toBe(13); }); test('bundle defaults: conservative off, balanced/tokenmax on @0.20', () => { @@ -585,6 +589,8 @@ describe('v0.42.3.0 — autocut knobs', () => { expect(MODE_BUNDLES.tokenmax.autocut).toBe(true); for (const m of ['conservative', 'balanced', 'tokenmax'] as const) { expect(MODE_BUNDLES[m].autocut_jump).toBe(0.2); + // v0.42.x — weak-top floor default 0.5 across all bundles. + expect(MODE_BUNDLES[m].autocut_min_top_score).toBe(0.5); } }); @@ -614,9 +620,37 @@ describe('v0.42.3.0 — autocut knobs', () => { expect(ov.autocut_jump).toBe(0.35); }); + test('resolveSearchMode threads autocut_min_top_score: per-call > config > bundle', () => { + expect(resolveSearchMode({ mode: 'balanced' }).autocut_min_top_score).toBe(0.5); + expect( + resolveSearchMode({ mode: 'balanced', overrides: { autocut_min_top_score: 0.7 } }).autocut_min_top_score, + ).toBe(0.7); + expect( + resolveSearchMode({ + mode: 'balanced', + overrides: { autocut_min_top_score: 0.7 }, + perCall: { autocut_min_top_score: 0.3 }, + }).autocut_min_top_score, + ).toBe(0.3); + }); + + test('loadOverridesFromConfig reads search.autocut_min_top_score (clamped [0,1])', () => { + expect(loadOverridesFromConfig({ 'search.autocut_min_top_score': '0.7' }).autocut_min_top_score).toBe(0.7); + expect(loadOverridesFromConfig({ 'search.autocut_min_top_score': '0' }).autocut_min_top_score).toBe(0); + // out of range → ignored (undefined, falls through to bundle) + expect(loadOverridesFromConfig({ 'search.autocut_min_top_score': '1.5' }).autocut_min_top_score).toBeUndefined(); + }); + test('SEARCH_MODE_CONFIG_KEYS includes the autocut keys', () => { expect(SEARCH_MODE_CONFIG_KEYS).toContain('search.autocut'); expect(SEARCH_MODE_CONFIG_KEYS).toContain('search.autocut_jump'); + expect(SEARCH_MODE_CONFIG_KEYS).toContain('search.autocut_min_top_score'); + }); + + test('knobsHash includes acmts= — different weak-top floors differ', () => { + const a = knobsHash(resolveSearchMode({ mode: 'balanced' })); // floor 0.5 + const b = knobsHash(resolveSearchMode({ mode: 'balanced', perCall: { autocut_min_top_score: 0.3 } })); + expect(a).not.toBe(b); }); test('knobsHash includes ac= / acj= — autocut-on vs off differ', () => { diff --git a/test/search/autocut-integration.serial.test.ts b/test/search/autocut-integration.serial.test.ts index ac1cdb51b..9e2481b1f 100644 --- a/test/search/autocut-integration.serial.test.ts +++ b/test/search/autocut-integration.serial.test.ts @@ -156,6 +156,53 @@ describe('autocut — ceiling override', () => { }); }); +describe('autocut — weak-top floor (cross-source collapse regression)', () => { + // The `--source __all__` collapse: on a rare-term cross-source query the top + // reranker score is WEAK (~0.317) yet, because autocut normalizes by the top, + // the normal decay below it looked like a confident cliff and trimmed a + // 32-result pool to 1. The weak-top floor (bundle default 0.5) no-ops autocut + // when the top isn't a confident anchor. Pinned end-to-end (the pure-fn tests + // can't prove it flows through resolveSearchMode → applyAutocut). + test('cliff after rank 1 but WEAK top (0.317) → no trim, recall preserved', async () => { + const baseline = await hybridSearch(engine, 'alpha keyword', { limit: 10 }); + // Exact live-brain shape that collapsed: 0.317 / 0.197 / 0.131 / 0.118. + const out = await hybridSearch(engine, 'alpha keyword', { + limit: 10, + reranker: rerankerOpts([0.317, 0.197, 0.131, 0.118, 0.1]), + }); + expect(out.length).toBe(baseline.length); + expect(out.length).toBeGreaterThanOrEqual(3); + }); + + test('same cliff shape but STRONG top (0.95) → trims to the confident answer', async () => { + // top 0.95 >= floor; normalized rank1→rank2 gap (1.0→0.62 = 0.38) cuts. + const out = await hybridSearch(engine, 'alpha keyword', { + limit: 10, + reranker: rerankerOpts([0.95, 0.59, 0.39, 0.35, 0.3]), + }); + expect(out.length).toBe(1); + }); + + test('per-call RAW-LOGIT reranker model → floor disabled, weak-top cliff still trims', async () => { + // The floor is gated on the EFFECTIVE reranker model. A per-call + // SearchOpts.reranker.model override to a raw-logit local reranker + // (llama-server/Qwen3) must disable the [0,1] floor — otherwise the weak + // 0.317 top would be wrongly read as "not confident" on an unbounded scale. + // With the floor off, the cliff (0.317→0.197) is trusted again → trims. + const out = await hybridSearch(engine, 'alpha keyword', { + limit: 10, + reranker: { + enabled: true, + topNIn: 30, + topNOut: null, + model: 'llama-server-reranker:Qwen3-Reranker-4B', + rerankerFn: rerankerWithScores([0.317, 0.197, 0.131, 0.118, 0.1]), + }, + }); + expect(out.length).toBe(1); + }); +}); + describe('autocut — composes with adaptive-return (never-empty holds)', () => { test('adaptive-return + autocut both on → non-empty, bounded', async () => { const out = await hybridSearch(engine, 'alpha keyword', { diff --git a/test/search/autocut.test.ts b/test/search/autocut.test.ts index 2b1cd2272..9b896dfc7 100644 --- a/test/search/autocut.test.ts +++ b/test/search/autocut.test.ts @@ -18,7 +18,7 @@ import { // items survived the cut. type R = { id: string; rs?: number }; const scoreOf = (r: R) => r.rs; -const ON: AutocutConfig = { enabled: true, jumpRatio: 0.2, minKeep: 1 }; +const ON: AutocutConfig = { enabled: true, jumpRatio: 0.2, minKeep: 1, minTopScore: 0.5 }; function mk(scores: Array): R[] { return scores.map((rs, i) => ({ id: `r${i}`, rs })); @@ -33,6 +33,12 @@ describe('DEFAULT_AUTOCUT', () => { test('is frozen', () => { expect(Object.isFrozen(DEFAULT_AUTOCUT)).toBe(true); }); + test('module default carries a weak-top floor (minTopScore 0.5)', () => { + // The reranker (zerank-2) is bimodal: real matches score ≈0.95+, weak + // matches ≈0.3. Below this floor the top is not a confident anchor, so + // normalizing by it manufactures a false cliff (see weak-top-floor block). + expect(DEFAULT_AUTOCUT.minTopScore).toBe(0.5); + }); }); describe('applyAutocut — cuts on a real cliff', () => { @@ -136,6 +142,56 @@ describe('applyAutocut — no-op guards', () => { }); }); +describe('applyAutocut — weak-top floor (cross-source collapse fix)', () => { + // Root cause of the `--source __all__` collapse: applyAutocut normalizes the + // cliff test by the top score, so a WEAK top (e.g. 0.317 on a rare-term + // cross-source query) gets rescaled to 1.0 and a normal decay below it looks + // like a confident cliff → the 32-result pool collapses to 1. Calibration on + // the live brain: every legitimate single-answer collapse has top_rerank≥0.95; + // the only bug case had top_rerank=0.317. minTopScore=0.5 sits in the empty + // middle of the reranker's bimodal distribution. + + test('weak top below floor → NO cut even when a cliff exists (recall preserved)', () => { + // The exact live-brain shape that collapsed: 0.317 / 0.197 / 0.131 / 0.118. + // Normalized, rank1→rank2 gap is 0.38 (clears jumpRatio 0.20) — but the top + // is a weak 0.317, so the cliff is untrustworthy. Keep all 4. + const r = applyAutocut(mk([0.317, 0.197, 0.131, 0.118]), scoreOf, ON); + expect(r.kept.length).toBe(4); + expect(r.decision.applied).toBe(false); + expect(r.decision.signal).toBe('none'); + }); + + test('strong top above floor → still cuts on a real cliff (precision preserved)', () => { + // The legitimate single-answer case: top 0.984 is a confident anchor and the + // cliff after rank 1 is real. The floor must NOT suppress this. + const r = applyAutocut(mk([0.984, 0.1, 0.08, 0.05]), scoreOf, ON); + expect(r.kept.map((x) => x.id)).toEqual(['r0']); + expect(r.decision.applied).toBe(true); + }); + + test('top exactly at the floor → treated as confident (>= floor cuts)', () => { + const r = applyAutocut(mk([0.5, 0.1, 0.05]), scoreOf, ON); + expect(r.kept.map((x) => x.id)).toEqual(['r0']); + expect(r.decision.applied).toBe(true); + }); + + test('minTopScore 0 disables the floor (explicit pre-fix behavior)', () => { + const r = applyAutocut(mk([0.317, 0.197, 0.131, 0.118]), scoreOf, { ...ON, minTopScore: 0 }); + expect(r.kept.length).toBe(1); + expect(r.decision.applied).toBe(true); + }); + + test('partial-knobs caller omitting minTopScore defaults to the 0.5 floor', () => { + // A literal that predates the field (or a JS caller) must compute the SAME + // trimmed set the knobsHash `acmts` default (0.5) claims — omitting the + // field must NOT silently disable the floor (compute/cache agreement). + const partial = { enabled: true, jumpRatio: 0.2, minKeep: 1 } as AutocutConfig; + const r = applyAutocut(mk([0.317, 0.197, 0.131, 0.118]), scoreOf, partial); + expect(r.kept.length).toBe(4); // weak top 0.317 < default 0.5 → no cut + expect(r.decision.applied).toBe(false); + }); +}); + describe('applyAutocut — failsafe', () => { test('never returns empty when input is non-empty', () => { const r = applyAutocut(mk([0.9, 0.01]), scoreOf, ON); @@ -208,6 +264,15 @@ describe('autocutFromConfig', () => { expect(autocutFromConfig({ search: { autocut_jump: 5 } }).jumpRatio).toBeUndefined(); expect(autocutFromConfig({ search: { autocut_jump: 0 } }).jumpRatio).toBeUndefined(); }); + test('reads search.autocut_min_top_score (clamped to [0, 1])', () => { + expect(autocutFromConfig({ search: { autocut_min_top_score: 0.7 } }).minTopScore).toBe(0.7); + // 0 is valid (disables the floor); 1 is valid (floor at the ceiling). + expect(autocutFromConfig({ search: { autocut_min_top_score: 0 } }).minTopScore).toBe(0); + expect(autocutFromConfig({ search: { autocut_min_top_score: 1 } }).minTopScore).toBe(1); + // Out of range → ignored (fall through to bundle/default). + expect(autocutFromConfig({ search: { autocut_min_top_score: -0.1 } }).minTopScore).toBeUndefined(); + expect(autocutFromConfig({ search: { autocut_min_top_score: 1.5 } }).minTopScore).toBeUndefined(); + }); test('empty / missing config → empty partial', () => { expect(autocutFromConfig(null)).toEqual({}); expect(autocutFromConfig({})).toEqual({}); @@ -229,4 +294,9 @@ describe('resolveAutocut — precedence ladder', () => { expect(cfg.jumpRatio).toBe(0.5); expect(cfg.enabled).toBe(false); // inherited from config (partial didn't set it) }); + test('minTopScore resolves default → config → per-call', () => { + expect(resolveAutocut(undefined, undefined).minTopScore).toBe(0.5); // module default + expect(resolveAutocut(undefined, { minTopScore: 0.7 }).minTopScore).toBe(0.7); // config + expect(resolveAutocut({ minTopScore: 0.3 }, { minTopScore: 0.7 }).minTopScore).toBe(0.3); // per-call wins + }); }); diff --git a/test/search/knobs-hash-reranker.test.ts b/test/search/knobs-hash-reranker.test.ts index 9f73ac394..914ca0744 100644 --- a/test/search/knobs-hash-reranker.test.ts +++ b/test/search/knobs-hash-reranker.test.ts @@ -44,7 +44,7 @@ function baseKnobs(): ResolvedSearchKnobs { } describe('KNOBS_HASH_VERSION + version invariants', () => { - test('version is 12 (…; 9→10 relational recall; 10→11 asymmetric input_type #1400; 11→12 hard-excludes #2825)', () => { + test('version is 13 (…; 10→11 asymmetric input_type #1400; 11→12 hard-excludes #2825; 12→13 autocut weak-top floor)', () => { // v0.35.0.0: 1→2 to fold reranker fields. v0.35.6.0: 2→3 to fold // floor_ratio. v0.36 wave: piggybacks on v=3 with 7 cross-modal knobs // (D2) PLUS column + provider context (D8/CDX-2 cross-column isolation). @@ -64,7 +64,9 @@ describe('KNOBS_HASH_VERSION + version invariants', () => { // pre-fix document-side query vectors must not be served. // #2825: 11→12 to fold the resolved hard-exclude prefix list (hx=) — // cached rows leaked GBRAIN_SEARCH_EXCLUDE'd slugs across processes. - expect(KNOBS_HASH_VERSION).toBe(12); + // 12→13: autocut weak-top floor (acmts=) — the floor shifts whether + // autocut cuts at all, so a different-floor write must not be served. + expect(KNOBS_HASH_VERSION).toBe(13); }); test('hash is 16 hex chars regardless of reranker config', () => { diff --git a/test/search/rerank.test.ts b/test/search/rerank.test.ts index 93f4e620f..b08a592d6 100644 --- a/test/search/rerank.test.ts +++ b/test/search/rerank.test.ts @@ -12,7 +12,7 @@ */ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; -import { applyReranker, type RerankerOpts } from '../../src/core/search/rerank.ts'; +import { applyReranker, rerankerEmitsNormalizedScores, type RerankerOpts } from '../../src/core/search/rerank.ts'; import { RerankError, type RerankResult } from '../../src/core/ai/gateway.ts'; import type { SearchResult } from '../../src/core/types.ts'; @@ -215,3 +215,22 @@ describe('applyReranker — pass-through cases', () => { expect(called).toBe(false); }); }); + +describe('rerankerEmitsNormalizedScores (autocut weak-top-floor gate)', () => { + test('cloud rerankers emit [0,1] scores → true', () => { + expect(rerankerEmitsNormalizedScores('zeroentropyai:zerank-2')).toBe(true); + expect(rerankerEmitsNormalizedScores('zeroentropyai:zerank-1-small')).toBe(true); + expect(rerankerEmitsNormalizedScores('cohere:rerank-v3.5')).toBe(true); + expect(rerankerEmitsNormalizedScores('voyage:rerank-2')).toBe(true); + }); + test('undefined (default ZE reranker) → true', () => { + expect(rerankerEmitsNormalizedScores(undefined)).toBe(true); + expect(rerankerEmitsNormalizedScores('')).toBe(true); + }); + test('raw-logit local rerankers (llama-server / Qwen3) → false', () => { + // These emit unbounded cross-encoder logits, so a [0,1] floor is meaningless. + expect(rerankerEmitsNormalizedScores('llama-server-reranker:Qwen3-Reranker-4B')).toBe(false); + expect(rerankerEmitsNormalizedScores('Qwen3-Reranker-0.6B')).toBe(false); + expect(rerankerEmitsNormalizedScores('LLAMA-SERVER-RERANKER:foo')).toBe(false); // case-insensitive + }); +});