fix(search): weak-top floor on autocut to stop cross-source result collapse

Takeover of #1863 (rebased onto master).

applyAutocut normalizes the cliff test by the top rerank score, so a weak
top (~0.317 on a rare-term cross-source pool) rescales to 1.0 and a normal
decay below it masquerades as a confident cliff — collapsing the pool to 1
result (`--source __all__` returned 1 while a single-source scoped query
returned ~12).

Fix: AutocutConfig.minTopScore (default 0.5) — below the floor autocut
no-ops and preserves recall; at/above it behavior is unchanged. Threaded
like autocut_jump (mode bundle → search.autocut_min_top_score config →
per-call → pick-chain). Gated on rerankerEmitsNormalizedScores(effective
model): raw-logit local rerankers (llama-server/Qwen3) get floor 0 since a
[0,1] threshold is meaningless on an unbounded logit scale.

Rebase deltas vs #1863: KNOBS_HASH_VERSION 12→13 (master took 12 for the
#2825 hard-exclude fold; acmts= folds into the key as the v=13 addition)
and version-pin tests/comments updated accordingly.

Co-authored-by: rayers <rayers@users.noreply.github.com>
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-07-21 14:39:44 -07:00
co-authored by rayers Claude Fable 5
parent 0612b0daa8
commit 8a0e956da3
12 changed files with 298 additions and 15 deletions
+1
View File
@@ -71,6 +71,7 @@ const KNOB_DESCRIPTIONS: Record<keyof ModeBundle, string> = {
// v0.42.3.0 autocut
autocut: 'Score-discontinuity result-sizing (cuts at the rerank-score cliff; no-op without a reranker)',
autocut_jump: 'Autocut sensitivity: min normalized score gap that counts as a cliff (0..1, 0.20 default)',
autocut_min_top_score: 'Autocut weak-top floor: min top rerank score to trust the cliff (0..1, 0.50 default; 0 disables; below it autocut no-ops to preserve recall)',
// v0.43 relational recall
relationalRetrieval: 'Typed-edge relational recall arm (relational queries walk the graph; no-op otherwise)',
relational_retrieval_depth: 'Max hops for relational traversal (1..3, 2 default)',
+32
View File
@@ -33,6 +33,19 @@ export interface AutocutConfig {
jumpRatio: number;
/** Failsafe: never return fewer than this when candidates exist (≥1). */
minKeep: number;
/**
* Weak-top floor: the minimum TOP rerank score for the cliff test to be
* trusted. The cliff is measured on scores NORMALIZED by the top, so a weak
* top (e.g. 0.317 on a rare-term cross-source query) rescales to 1.0 and a
* normal decay below it masquerades as a confident cliff — collapsing a rich
* pool to 1. When the top score is below this floor, autocut no-ops (recall
* preserved); above it, the cliff is trusted as before. The reranker
* (zerank-2) is bimodal — real matches ≈0.95+, weak matches ≈0.3 — so the
* default 0.5 sits in the empty middle. 0 disables the floor (pre-fix
* behavior). Clamped to [0, 1]. Override: `search.autocut_min_top_score`
* config → mode bundle.
*/
minTopScore: number;
}
/**
@@ -44,6 +57,8 @@ export const DEFAULT_AUTOCUT: AutocutConfig = Object.freeze({
enabled: true,
jumpRatio: 0.2,
minKeep: 1,
// Weak-top floor (cross-source collapse fix). See AutocutConfig.minTopScore.
minTopScore: 0.5,
});
export interface AutocutDecision {
@@ -79,6 +94,13 @@ export function autocutFromConfig(
typeof search.autocut_min_keep === 'number' ? Math.floor(search.autocut_min_keep) : Number.NaN;
if (Number.isFinite(n) && n >= 1) out.minKeep = n;
}
if (search.autocut_min_top_score !== undefined) {
const n =
typeof search.autocut_min_top_score === 'number' ? search.autocut_min_top_score : Number.NaN;
// [0, 1]: 0 disables the floor, 1 pins it at the ceiling. Out-of-range
// values are IGNORED (fall through to bundle / module default).
if (Number.isFinite(n) && n >= 0 && n <= 1) out.minTopScore = n;
}
return out;
}
@@ -160,6 +182,16 @@ export function applyAutocut<T>(
const top = Math.max(...scores);
if (!Number.isFinite(top) || top <= 0) return noOp(results);
// Weak-top floor: the cliff is measured on scores normalized by `top`, so a
// weak top fabricates a confident-looking cliff (rare-term cross-source
// queries collapse 32→1). When the top isn't a confident anchor, don't trust
// the cliff — no-op and preserve recall. `?? 0.5` mirrors DEFAULT_AUTOCUT and
// the knobsHash `acmts` default so a partial-knobs literal caller computes
// the same trimmed set the cache key claims it did (omitting the field must
// not silently disable the floor). Pass minTopScore:0 to disable explicitly.
const minTopScore = cfg.minTopScore ?? 0.5;
if (top < minTopScore) return noOp(results);
// Sort a copy descending (A2: don't trust upstream order) and normalize.
const sorted = [...scores].sort((a, b) => b - a);
const norm = sorted.map((s) => s / top);
+15 -2
View File
@@ -27,7 +27,7 @@ import { applyAutocut, type AutocutDecision } from './autocut.ts';
import { buildRelationalArm } from './relational-recall.ts';
import { loadConfigWithEngine } from '../config.ts';
import { dedupResults } from './dedup.ts';
import { applyReranker } from './rerank.ts';
import { applyReranker, rerankerEmitsNormalizedScores } from './rerank.ts';
import { autoDetectDetail, classifyQuery, isAmbiguousModalityQuery } from './query-intent.ts';
import { isTitlePhraseMatch } from './title-match.ts';
import { normalizeAlias } from './alias-normalize.ts';
@@ -1572,10 +1572,23 @@ export async function hybridSearch(
// never-empty failsafe (1); jumpRatio comes from the resolved mode.
let autocutDecision: AutocutDecision | undefined;
if (resolvedMode.autocut && offset === 0) {
// v0.42.x — minTopScore: weak-top floor. When the top rerank score isn't a
// confident anchor, autocut no-ops (fixes the rare-term cross-source
// collapse where a weak 0.317 top fabricated a cliff and trimmed 32→1). The
// floor is a [0,1] threshold, so it only applies to rerankers that emit
// normalized scores; on raw-logit rerankers (llama-server/Qwen3) we pass 0
// (disabled) to avoid suppressing legitimate cuts on an unbounded scale.
// Gate on the EFFECTIVE reranker model — a per-call SearchOpts.reranker
// .model override wins over the bundle default, so reading the bundle alone
// would re-open the bug for a per-call raw-logit reranker.
const effectiveRerankerModel = rerankerOpts.model ?? resolvedMode.reranker_model;
const autocutMinTopScore = rerankerEmitsNormalizedScores(effectiveRerankerModel)
? resolvedMode.autocut_min_top_score
: 0;
const r = applyAutocut(
returnPool,
(x) => x.rerank_score,
{ enabled: true, jumpRatio: resolvedMode.autocut_jump, minKeep: 1 },
{ enabled: true, jumpRatio: resolvedMode.autocut_jump, minKeep: 1, minTopScore: autocutMinTopScore },
// Preserve alias-hop exact matches: applyAliasHop injects the canonical
// page AFTER reranking, so it has no rerank_score. Without this it would
// be dropped whenever autocut cuts on the scored set (Codex P1).
+41 -1
View File
@@ -273,6 +273,16 @@ export interface ModeBundle {
relationalRetrieval: boolean;
/** v0.43 — max hops for relational traversal. Default 2, hard-capped at 3. */
relational_retrieval_depth: number;
/**
* v0.42.x — autocut weak-top floor: the minimum top cross-encoder rerank
* score for the cliff cut to be trusted. autocut normalizes the cliff test by
* the top score, so a weak top (e.g. 0.317 on a rare-term cross-source query)
* rescales to 1.0 and fabricates a confident cliff — collapsing a rich pool to
* 1 (the `--source __all__` collapse). Below this floor autocut no-ops (recall
* preserved). Default 0.5 (zerank-2 is bimodal: real ≈0.95+, weak ≈0.3). 0
* disables the floor. Override: `search.autocut_min_top_score` config → bundle.
*/
autocut_min_top_score: number;
}
/**
@@ -326,6 +336,7 @@ export const MODE_BUNDLES: Readonly<Record<SearchMode, Readonly<ModeBundle>>> =
relationalRetrieval: false,
relational_retrieval_depth: 2,
autocut_jump: 0.2,
autocut_min_top_score: 0.5,
}),
balanced: Object.freeze({
cache_enabled: true,
@@ -384,6 +395,7 @@ export const MODE_BUNDLES: Readonly<Record<SearchMode, Readonly<ModeBundle>>> =
relationalRetrieval: true,
relational_retrieval_depth: 2,
autocut_jump: 0.2,
autocut_min_top_score: 0.5,
}),
tokenmax: Object.freeze({
cache_enabled: true,
@@ -435,6 +447,7 @@ export const MODE_BUNDLES: Readonly<Record<SearchMode, Readonly<ModeBundle>>> =
relationalRetrieval: true,
relational_retrieval_depth: 2,
autocut_jump: 0.2,
autocut_min_top_score: 0.5,
}),
});
@@ -489,6 +502,8 @@ export interface SearchKeyOverrides {
relationalRetrieval?: boolean;
relational_retrieval_depth?: number;
autocut_jump?: number;
// v0.42.x — autocut weak-top floor override.
autocut_min_top_score?: number;
}
/**
@@ -538,6 +553,8 @@ export interface SearchPerCallOpts {
// v0.43 — relational recall per-call overrides.
relationalRetrieval?: boolean;
relational_retrieval_depth?: number;
// v0.42.x — autocut weak-top floor per-call override.
autocut_min_top_score?: number;
}
/**
@@ -633,6 +650,8 @@ export function resolveSearchMode(input: ResolveSearchModeInput): ResolvedSearch
// v0.43 — relational recall resolved via the same pick chain.
relationalRetrieval: pick('relationalRetrieval'),
relational_retrieval_depth: pick('relational_retrieval_depth'),
// v0.42.x — autocut weak-top floor resolved via the same pick chain.
autocut_min_top_score: pick('autocut_min_top_score'),
resolved_mode,
mode_valid: valid,
};
@@ -756,7 +775,12 @@ export function attributeKnob<K extends keyof ModeBundle>(
// slugs written by a process without it, and vice versa. Same one-time
// global cold-miss pattern as the bumps above; refills within
// cache.ttl_seconds (3600s default).
export const KNOBS_HASH_VERSION = 12;
//
// bump 12→13 (autocut weak-top floor): adds `acmts` (autocut_min_top_score).
// The floor changes WHETHER autocut cuts at all — a write made with one floor
// must NOT be served to a lookup at a different floor (the trimmed-vs-full set
// differs). Same one-time global cold-miss pattern; fills within cache.ttl.
export const KNOBS_HASH_VERSION = 13;
/**
* v0.36 (D8 / CDX-2) — second-arg context for the cache key. The
@@ -888,6 +912,13 @@ export function knobsHash(
// across processes. Sorted copy so ['a/','b/'] and ['b/','a/'] hash
// identically; undefined falls back to 'none' for legacy callers.
`hx=${ctx?.hardExcludes ? [...ctx.hardExcludes].sort().join(',') : 'none'}`,
// v=13 addition (append-only) — autocut weak-top floor shifts whether
// autocut cuts at all, so an autocut-cut write must not be served to a
// different-floor lookup. `?? 0.5` mirrors the module default for
// partial-knobs callers. 4 decimals (vs acj's 2): the floor is compared
// directly against raw rerank scores, so nearby config values (0.501 vs
// 0.504) can flip trim-vs-no-op and must not collide on the cache key.
`acmts=${(knobs.autocut_min_top_score ?? 0.5).toFixed(4)}`,
];
const h = createHash('sha256');
h.update(parts.join('|'));
@@ -1054,6 +1085,13 @@ export function loadOverridesFromConfig(
const n = parseFloat(acj);
if (Number.isFinite(n) && n > 0 && n <= 1) out.autocut_jump = n;
}
// v0.42.x — autocut weak-top floor. [0, 1]: 0 disables, 1 pins at ceiling;
// out-of-range falls through to the bundle.
const acmts = get('search.autocut_min_top_score');
if (acmts !== undefined) {
const n = parseFloat(acmts);
if (Number.isFinite(n) && n >= 0 && n <= 1) out.autocut_min_top_score = n;
}
// v0.43 — relational recall arm.
const rel = get('search.relational_retrieval');
@@ -1108,6 +1146,8 @@ export const SEARCH_MODE_CONFIG_KEYS: ReadonlyArray<string> = Object.freeze([
'search.relational_retrieval',
'search.relational_retrieval_depth',
'search.autocut_jump',
// v0.42.x autocut weak-top floor
'search.autocut_min_top_score',
]);
/**
+24
View File
@@ -39,6 +39,30 @@ export interface RerankerOpts {
rerankerFn?: (input: RerankInput) => Promise<RerankResult[]>;
}
/**
* Does this reranker emit relevance scores normalized to roughly [0, 1]?
*
* `rerank_score` is taken verbatim from the provider response (gateway.ts —
* no sigmoid, no normalization). Cloud rerankers (ZeroEntropy zerank-*, Cohere,
* Voyage) return calibrated [0, 1] relevance. But the self-hosted
* `llama-server-reranker` path (Qwen3-Reranker GGUF via llama.cpp `--reranking`)
* returns the RAW cross-encoder logit — unbounded, frequently negative for
* non-matches and well above 1 for strong matches.
*
* This matters for autocut's weak-top floor (autocut.ts minTopScore): the floor
* is a [0, 1] threshold and is meaningless on a raw-logit scale, where it would
* either suppress legitimate cuts (a confident logit that happens to land in
* (0, floor)) or no-op entirely (logits ≫ 1). Callers gate the floor on this.
*
* Conservative: returns false ONLY for the known raw-logit local rerankers, so
* an unrecognized cloud model still gets the (correct) normalized treatment.
*/
export function rerankerEmitsNormalizedScores(model?: string): boolean {
if (!model) return true; // default reranker is ZE zerank-2 ([0,1])
const m = model.toLowerCase();
return !(m.includes('llama-server-reranker') || m.includes('qwen3-reranker'));
}
/** SHA-256 prefix (8 chars) of the query text for privacy-preserving audit. */
function hashQuery(query: string): string {
return createHash('sha256').update(query, 'utf8').digest('hex').slice(0, 8);
+3 -2
View File
@@ -136,7 +136,7 @@ describe('D2 — knobsHash differs across cross-modal knob values', () => {
return resolveSearchMode({ mode: 'balanced' });
}
test('KNOBS_HASH_VERSION is 12 (cross-modal still appended; 11→12 hard-exclude fold #2825)', () => {
test('KNOBS_HASH_VERSION is 13 (cross-modal still appended; 12→13 autocut weak-top floor)', () => {
// v0.35 ladder: 1→2 reranker, 2→3 floor_ratio. v0.36 piggybacks on v=3
// with 7 cross-modal knobs + column/provider context. v0.40.4 (salem) +
// v0.39 T21 (master) bump to v=4 for graph_signals + schema-pack fields.
@@ -146,7 +146,8 @@ describe('D2 — knobsHash differs across cross-modal knob values', () => {
// v0.43: 9→10 relational recall arm. #1400: 10→11 query-side input_type
// finally reaches asymmetric providers — pre-fix rows were keyed on
// document-side query vectors. #2825: 11→12 hard-exclude fold (hx=).
expect(KNOBS_HASH_VERSION).toBe(12);
// 12→13: autocut weak-top floor (acmts=).
expect(KNOBS_HASH_VERSION).toBe(13);
});
test('flipping unified_multimodal changes the hash', () => {
+2 -2
View File
@@ -89,7 +89,7 @@ describe('alias_resolved boost stage', () => {
});
describe('KNOBS_HASH_VERSION', () => {
it('is 12 (11→12 hard-exclude fold invalidates rows written under a different exclude policy, #2825)', () => {
expect(KNOBS_HASH_VERSION).toBe(12);
it('is 13 (12→13 autocut weak-top floor acmts= added; 11→12 was #2825 hard-exclude fold)', () => {
expect(KNOBS_HASH_VERSION).toBe(13);
});
});
+38 -4
View File
@@ -80,6 +80,7 @@ describe('SEARCH_MODES + MODE_BUNDLES canonical shape', () => {
// v0.43 — relational recall OFF for conservative.
relationalRetrieval: false,
relational_retrieval_depth: 2,
autocut_min_top_score: 0.5,
});
});
@@ -112,6 +113,7 @@ describe('SEARCH_MODES + MODE_BUNDLES canonical shape', () => {
// v0.43 — relational recall ON for balanced.
relationalRetrieval: true,
relational_retrieval_depth: 2,
autocut_min_top_score: 0.5,
});
});
@@ -142,6 +144,7 @@ describe('SEARCH_MODES + MODE_BUNDLES canonical shape', () => {
// v0.43 — relational recall ON for tokenmax.
relationalRetrieval: true,
relational_retrieval_depth: 2,
autocut_min_top_score: 0.5,
});
});
@@ -409,8 +412,9 @@ describe('knobsHash determinism + cross-mode separation (CDX-4)', () => {
// must not be served to post-fix lookups.
// #2825: bumped 11→12 to fold the resolved hard-exclude prefix list
// (hx=) — cached rows leaked GBRAIN_SEARCH_EXCLUDE'd slugs across
// processes.
expect(KNOBS_HASH_VERSION).toBe(12);
// processes. 12→13: autocut weak-top floor (acmts=) — the floor shifts
// whether autocut cuts at all.
expect(KNOBS_HASH_VERSION).toBe(13);
});
test('T1 (codex): floor_ratio set vs unset produces DIFFERENT hashes (cache contamination prevention)', () => {
@@ -575,8 +579,8 @@ describe('v0.40.4 — graph_signals knob', () => {
});
describe('v0.42.3.0 — autocut knobs', () => {
test('KNOBS_HASH_VERSION is 12 (11→12 hard-exclude fold, #2825)', () => {
expect(KNOBS_HASH_VERSION).toBe(12);
test('KNOBS_HASH_VERSION is 13 (12→13 autocut weak-top floor; 11→12 was #2825 hard-excludes)', () => {
expect(KNOBS_HASH_VERSION).toBe(13);
});
test('bundle defaults: conservative off, balanced/tokenmax on @0.20', () => {
@@ -585,6 +589,8 @@ describe('v0.42.3.0 — autocut knobs', () => {
expect(MODE_BUNDLES.tokenmax.autocut).toBe(true);
for (const m of ['conservative', 'balanced', 'tokenmax'] as const) {
expect(MODE_BUNDLES[m].autocut_jump).toBe(0.2);
// v0.42.x — weak-top floor default 0.5 across all bundles.
expect(MODE_BUNDLES[m].autocut_min_top_score).toBe(0.5);
}
});
@@ -614,9 +620,37 @@ describe('v0.42.3.0 — autocut knobs', () => {
expect(ov.autocut_jump).toBe(0.35);
});
test('resolveSearchMode threads autocut_min_top_score: per-call > config > bundle', () => {
expect(resolveSearchMode({ mode: 'balanced' }).autocut_min_top_score).toBe(0.5);
expect(
resolveSearchMode({ mode: 'balanced', overrides: { autocut_min_top_score: 0.7 } }).autocut_min_top_score,
).toBe(0.7);
expect(
resolveSearchMode({
mode: 'balanced',
overrides: { autocut_min_top_score: 0.7 },
perCall: { autocut_min_top_score: 0.3 },
}).autocut_min_top_score,
).toBe(0.3);
});
test('loadOverridesFromConfig reads search.autocut_min_top_score (clamped [0,1])', () => {
expect(loadOverridesFromConfig({ 'search.autocut_min_top_score': '0.7' }).autocut_min_top_score).toBe(0.7);
expect(loadOverridesFromConfig({ 'search.autocut_min_top_score': '0' }).autocut_min_top_score).toBe(0);
// out of range → ignored (undefined, falls through to bundle)
expect(loadOverridesFromConfig({ 'search.autocut_min_top_score': '1.5' }).autocut_min_top_score).toBeUndefined();
});
test('SEARCH_MODE_CONFIG_KEYS includes the autocut keys', () => {
expect(SEARCH_MODE_CONFIG_KEYS).toContain('search.autocut');
expect(SEARCH_MODE_CONFIG_KEYS).toContain('search.autocut_jump');
expect(SEARCH_MODE_CONFIG_KEYS).toContain('search.autocut_min_top_score');
});
test('knobsHash includes acmts= — different weak-top floors differ', () => {
const a = knobsHash(resolveSearchMode({ mode: 'balanced' })); // floor 0.5
const b = knobsHash(resolveSearchMode({ mode: 'balanced', perCall: { autocut_min_top_score: 0.3 } }));
expect(a).not.toBe(b);
});
test('knobsHash includes ac= / acj= — autocut-on vs off differ', () => {
@@ -156,6 +156,53 @@ describe('autocut — ceiling override', () => {
});
});
describe('autocut — weak-top floor (cross-source collapse regression)', () => {
// The `--source __all__` collapse: on a rare-term cross-source query the top
// reranker score is WEAK (~0.317) yet, because autocut normalizes by the top,
// the normal decay below it looked like a confident cliff and trimmed a
// 32-result pool to 1. The weak-top floor (bundle default 0.5) no-ops autocut
// when the top isn't a confident anchor. Pinned end-to-end (the pure-fn tests
// can't prove it flows through resolveSearchMode → applyAutocut).
test('cliff after rank 1 but WEAK top (0.317) → no trim, recall preserved', async () => {
const baseline = await hybridSearch(engine, 'alpha keyword', { limit: 10 });
// Exact live-brain shape that collapsed: 0.317 / 0.197 / 0.131 / 0.118.
const out = await hybridSearch(engine, 'alpha keyword', {
limit: 10,
reranker: rerankerOpts([0.317, 0.197, 0.131, 0.118, 0.1]),
});
expect(out.length).toBe(baseline.length);
expect(out.length).toBeGreaterThanOrEqual(3);
});
test('same cliff shape but STRONG top (0.95) → trims to the confident answer', async () => {
// top 0.95 >= floor; normalized rank1→rank2 gap (1.0→0.62 = 0.38) cuts.
const out = await hybridSearch(engine, 'alpha keyword', {
limit: 10,
reranker: rerankerOpts([0.95, 0.59, 0.39, 0.35, 0.3]),
});
expect(out.length).toBe(1);
});
test('per-call RAW-LOGIT reranker model → floor disabled, weak-top cliff still trims', async () => {
// The floor is gated on the EFFECTIVE reranker model. A per-call
// SearchOpts.reranker.model override to a raw-logit local reranker
// (llama-server/Qwen3) must disable the [0,1] floor — otherwise the weak
// 0.317 top would be wrongly read as "not confident" on an unbounded scale.
// With the floor off, the cliff (0.317→0.197) is trusted again → trims.
const out = await hybridSearch(engine, 'alpha keyword', {
limit: 10,
reranker: {
enabled: true,
topNIn: 30,
topNOut: null,
model: 'llama-server-reranker:Qwen3-Reranker-4B',
rerankerFn: rerankerWithScores([0.317, 0.197, 0.131, 0.118, 0.1]),
},
});
expect(out.length).toBe(1);
});
});
describe('autocut — composes with adaptive-return (never-empty holds)', () => {
test('adaptive-return + autocut both on → non-empty, bounded', async () => {
const out = await hybridSearch(engine, 'alpha keyword', {
+71 -1
View File
@@ -18,7 +18,7 @@ import {
// items survived the cut.
type R = { id: string; rs?: number };
const scoreOf = (r: R) => r.rs;
const ON: AutocutConfig = { enabled: true, jumpRatio: 0.2, minKeep: 1 };
const ON: AutocutConfig = { enabled: true, jumpRatio: 0.2, minKeep: 1, minTopScore: 0.5 };
function mk(scores: Array<number | undefined>): R[] {
return scores.map((rs, i) => ({ id: `r${i}`, rs }));
@@ -33,6 +33,12 @@ describe('DEFAULT_AUTOCUT', () => {
test('is frozen', () => {
expect(Object.isFrozen(DEFAULT_AUTOCUT)).toBe(true);
});
test('module default carries a weak-top floor (minTopScore 0.5)', () => {
// The reranker (zerank-2) is bimodal: real matches score ≈0.95+, weak
// matches ≈0.3. Below this floor the top is not a confident anchor, so
// normalizing by it manufactures a false cliff (see weak-top-floor block).
expect(DEFAULT_AUTOCUT.minTopScore).toBe(0.5);
});
});
describe('applyAutocut — cuts on a real cliff', () => {
@@ -136,6 +142,56 @@ describe('applyAutocut — no-op guards', () => {
});
});
describe('applyAutocut — weak-top floor (cross-source collapse fix)', () => {
// Root cause of the `--source __all__` collapse: applyAutocut normalizes the
// cliff test by the top score, so a WEAK top (e.g. 0.317 on a rare-term
// cross-source query) gets rescaled to 1.0 and a normal decay below it looks
// like a confident cliff → the 32-result pool collapses to 1. Calibration on
// the live brain: every legitimate single-answer collapse has top_rerank≥0.95;
// the only bug case had top_rerank=0.317. minTopScore=0.5 sits in the empty
// middle of the reranker's bimodal distribution.
test('weak top below floor → NO cut even when a cliff exists (recall preserved)', () => {
// The exact live-brain shape that collapsed: 0.317 / 0.197 / 0.131 / 0.118.
// Normalized, rank1→rank2 gap is 0.38 (clears jumpRatio 0.20) — but the top
// is a weak 0.317, so the cliff is untrustworthy. Keep all 4.
const r = applyAutocut(mk([0.317, 0.197, 0.131, 0.118]), scoreOf, ON);
expect(r.kept.length).toBe(4);
expect(r.decision.applied).toBe(false);
expect(r.decision.signal).toBe('none');
});
test('strong top above floor → still cuts on a real cliff (precision preserved)', () => {
// The legitimate single-answer case: top 0.984 is a confident anchor and the
// cliff after rank 1 is real. The floor must NOT suppress this.
const r = applyAutocut(mk([0.984, 0.1, 0.08, 0.05]), scoreOf, ON);
expect(r.kept.map((x) => x.id)).toEqual(['r0']);
expect(r.decision.applied).toBe(true);
});
test('top exactly at the floor → treated as confident (>= floor cuts)', () => {
const r = applyAutocut(mk([0.5, 0.1, 0.05]), scoreOf, ON);
expect(r.kept.map((x) => x.id)).toEqual(['r0']);
expect(r.decision.applied).toBe(true);
});
test('minTopScore 0 disables the floor (explicit pre-fix behavior)', () => {
const r = applyAutocut(mk([0.317, 0.197, 0.131, 0.118]), scoreOf, { ...ON, minTopScore: 0 });
expect(r.kept.length).toBe(1);
expect(r.decision.applied).toBe(true);
});
test('partial-knobs caller omitting minTopScore defaults to the 0.5 floor', () => {
// A literal that predates the field (or a JS caller) must compute the SAME
// trimmed set the knobsHash `acmts` default (0.5) claims — omitting the
// field must NOT silently disable the floor (compute/cache agreement).
const partial = { enabled: true, jumpRatio: 0.2, minKeep: 1 } as AutocutConfig;
const r = applyAutocut(mk([0.317, 0.197, 0.131, 0.118]), scoreOf, partial);
expect(r.kept.length).toBe(4); // weak top 0.317 < default 0.5 → no cut
expect(r.decision.applied).toBe(false);
});
});
describe('applyAutocut — failsafe', () => {
test('never returns empty when input is non-empty', () => {
const r = applyAutocut(mk([0.9, 0.01]), scoreOf, ON);
@@ -208,6 +264,15 @@ describe('autocutFromConfig', () => {
expect(autocutFromConfig({ search: { autocut_jump: 5 } }).jumpRatio).toBeUndefined();
expect(autocutFromConfig({ search: { autocut_jump: 0 } }).jumpRatio).toBeUndefined();
});
test('reads search.autocut_min_top_score (clamped to [0, 1])', () => {
expect(autocutFromConfig({ search: { autocut_min_top_score: 0.7 } }).minTopScore).toBe(0.7);
// 0 is valid (disables the floor); 1 is valid (floor at the ceiling).
expect(autocutFromConfig({ search: { autocut_min_top_score: 0 } }).minTopScore).toBe(0);
expect(autocutFromConfig({ search: { autocut_min_top_score: 1 } }).minTopScore).toBe(1);
// Out of range → ignored (fall through to bundle/default).
expect(autocutFromConfig({ search: { autocut_min_top_score: -0.1 } }).minTopScore).toBeUndefined();
expect(autocutFromConfig({ search: { autocut_min_top_score: 1.5 } }).minTopScore).toBeUndefined();
});
test('empty / missing config → empty partial', () => {
expect(autocutFromConfig(null)).toEqual({});
expect(autocutFromConfig({})).toEqual({});
@@ -229,4 +294,9 @@ describe('resolveAutocut — precedence ladder', () => {
expect(cfg.jumpRatio).toBe(0.5);
expect(cfg.enabled).toBe(false); // inherited from config (partial didn't set it)
});
test('minTopScore resolves default → config → per-call', () => {
expect(resolveAutocut(undefined, undefined).minTopScore).toBe(0.5); // module default
expect(resolveAutocut(undefined, { minTopScore: 0.7 }).minTopScore).toBe(0.7); // config
expect(resolveAutocut({ minTopScore: 0.3 }, { minTopScore: 0.7 }).minTopScore).toBe(0.3); // per-call wins
});
});
+4 -2
View File
@@ -44,7 +44,7 @@ function baseKnobs(): ResolvedSearchKnobs {
}
describe('KNOBS_HASH_VERSION + version invariants', () => {
test('version is 12 (…; 9→10 relational recall; 10→11 asymmetric input_type #1400; 11→12 hard-excludes #2825)', () => {
test('version is 13 (…; 10→11 asymmetric input_type #1400; 11→12 hard-excludes #2825; 12→13 autocut weak-top floor)', () => {
// v0.35.0.0: 1→2 to fold reranker fields. v0.35.6.0: 2→3 to fold
// floor_ratio. v0.36 wave: piggybacks on v=3 with 7 cross-modal knobs
// (D2) PLUS column + provider context (D8/CDX-2 cross-column isolation).
@@ -64,7 +64,9 @@ describe('KNOBS_HASH_VERSION + version invariants', () => {
// pre-fix document-side query vectors must not be served.
// #2825: 11→12 to fold the resolved hard-exclude prefix list (hx=) —
// cached rows leaked GBRAIN_SEARCH_EXCLUDE'd slugs across processes.
expect(KNOBS_HASH_VERSION).toBe(12);
// 12→13: autocut weak-top floor (acmts=) — the floor shifts whether
// autocut cuts at all, so a different-floor write must not be served.
expect(KNOBS_HASH_VERSION).toBe(13);
});
test('hash is 16 hex chars regardless of reranker config', () => {
+20 -1
View File
@@ -12,7 +12,7 @@
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { applyReranker, type RerankerOpts } from '../../src/core/search/rerank.ts';
import { applyReranker, rerankerEmitsNormalizedScores, type RerankerOpts } from '../../src/core/search/rerank.ts';
import { RerankError, type RerankResult } from '../../src/core/ai/gateway.ts';
import type { SearchResult } from '../../src/core/types.ts';
@@ -215,3 +215,22 @@ describe('applyReranker — pass-through cases', () => {
expect(called).toBe(false);
});
});
describe('rerankerEmitsNormalizedScores (autocut weak-top-floor gate)', () => {
test('cloud rerankers emit [0,1] scores → true', () => {
expect(rerankerEmitsNormalizedScores('zeroentropyai:zerank-2')).toBe(true);
expect(rerankerEmitsNormalizedScores('zeroentropyai:zerank-1-small')).toBe(true);
expect(rerankerEmitsNormalizedScores('cohere:rerank-v3.5')).toBe(true);
expect(rerankerEmitsNormalizedScores('voyage:rerank-2')).toBe(true);
});
test('undefined (default ZE reranker) → true', () => {
expect(rerankerEmitsNormalizedScores(undefined)).toBe(true);
expect(rerankerEmitsNormalizedScores('')).toBe(true);
});
test('raw-logit local rerankers (llama-server / Qwen3) → false', () => {
// These emit unbounded cross-encoder logits, so a [0,1] floor is meaningless.
expect(rerankerEmitsNormalizedScores('llama-server-reranker:Qwen3-Reranker-4B')).toBe(false);
expect(rerankerEmitsNormalizedScores('Qwen3-Reranker-0.6B')).toBe(false);
expect(rerankerEmitsNormalizedScores('LLAMA-SERVER-RERANKER:foo')).toBe(false); // case-insensitive
});
});