mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-31 04:07:52 +00:00
fix(search): weak-top floor on autocut to stop cross-source result collapse
Takeover of #1863 (rebased onto master). applyAutocut normalizes the cliff test by the top rerank score, so a weak top (~0.317 on a rare-term cross-source pool) rescales to 1.0 and a normal decay below it masquerades as a confident cliff — collapsing the pool to 1 result (`--source __all__` returned 1 while a single-source scoped query returned ~12). Fix: AutocutConfig.minTopScore (default 0.5) — below the floor autocut no-ops and preserves recall; at/above it behavior is unchanged. Threaded like autocut_jump (mode bundle → search.autocut_min_top_score config → per-call → pick-chain). Gated on rerankerEmitsNormalizedScores(effective model): raw-logit local rerankers (llama-server/Qwen3) get floor 0 since a [0,1] threshold is meaningless on an unbounded logit scale. Rebase deltas vs #1863: KNOBS_HASH_VERSION 12→13 (master took 12 for the #2825 hard-exclude fold; acmts= folds into the key as the v=13 addition) and version-pin tests/comments updated accordingly. Co-authored-by: rayers <rayers@users.noreply.github.com> Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
rayers
Claude Fable 5
parent
0612b0daa8
commit
8a0e956da3
@@ -71,6 +71,7 @@ const KNOB_DESCRIPTIONS: Record<keyof ModeBundle, string> = {
|
||||
// v0.42.3.0 autocut
|
||||
autocut: 'Score-discontinuity result-sizing (cuts at the rerank-score cliff; no-op without a reranker)',
|
||||
autocut_jump: 'Autocut sensitivity: min normalized score gap that counts as a cliff (0..1, 0.20 default)',
|
||||
autocut_min_top_score: 'Autocut weak-top floor: min top rerank score to trust the cliff (0..1, 0.50 default; 0 disables; below it autocut no-ops to preserve recall)',
|
||||
// v0.43 relational recall
|
||||
relationalRetrieval: 'Typed-edge relational recall arm (relational queries walk the graph; no-op otherwise)',
|
||||
relational_retrieval_depth: 'Max hops for relational traversal (1..3, 2 default)',
|
||||
|
||||
@@ -33,6 +33,19 @@ export interface AutocutConfig {
|
||||
jumpRatio: number;
|
||||
/** Failsafe: never return fewer than this when candidates exist (≥1). */
|
||||
minKeep: number;
|
||||
/**
|
||||
* Weak-top floor: the minimum TOP rerank score for the cliff test to be
|
||||
* trusted. The cliff is measured on scores NORMALIZED by the top, so a weak
|
||||
* top (e.g. 0.317 on a rare-term cross-source query) rescales to 1.0 and a
|
||||
* normal decay below it masquerades as a confident cliff — collapsing a rich
|
||||
* pool to 1. When the top score is below this floor, autocut no-ops (recall
|
||||
* preserved); above it, the cliff is trusted as before. The reranker
|
||||
* (zerank-2) is bimodal — real matches ≈0.95+, weak matches ≈0.3 — so the
|
||||
* default 0.5 sits in the empty middle. 0 disables the floor (pre-fix
|
||||
* behavior). Clamped to [0, 1]. Override: `search.autocut_min_top_score`
|
||||
* config → mode bundle.
|
||||
*/
|
||||
minTopScore: number;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -44,6 +57,8 @@ export const DEFAULT_AUTOCUT: AutocutConfig = Object.freeze({
|
||||
enabled: true,
|
||||
jumpRatio: 0.2,
|
||||
minKeep: 1,
|
||||
// Weak-top floor (cross-source collapse fix). See AutocutConfig.minTopScore.
|
||||
minTopScore: 0.5,
|
||||
});
|
||||
|
||||
export interface AutocutDecision {
|
||||
@@ -79,6 +94,13 @@ export function autocutFromConfig(
|
||||
typeof search.autocut_min_keep === 'number' ? Math.floor(search.autocut_min_keep) : Number.NaN;
|
||||
if (Number.isFinite(n) && n >= 1) out.minKeep = n;
|
||||
}
|
||||
if (search.autocut_min_top_score !== undefined) {
|
||||
const n =
|
||||
typeof search.autocut_min_top_score === 'number' ? search.autocut_min_top_score : Number.NaN;
|
||||
// [0, 1]: 0 disables the floor, 1 pins it at the ceiling. Out-of-range
|
||||
// values are IGNORED (fall through to bundle / module default).
|
||||
if (Number.isFinite(n) && n >= 0 && n <= 1) out.minTopScore = n;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
@@ -160,6 +182,16 @@ export function applyAutocut<T>(
|
||||
const top = Math.max(...scores);
|
||||
if (!Number.isFinite(top) || top <= 0) return noOp(results);
|
||||
|
||||
// Weak-top floor: the cliff is measured on scores normalized by `top`, so a
|
||||
// weak top fabricates a confident-looking cliff (rare-term cross-source
|
||||
// queries collapse 32→1). When the top isn't a confident anchor, don't trust
|
||||
// the cliff — no-op and preserve recall. `?? 0.5` mirrors DEFAULT_AUTOCUT and
|
||||
// the knobsHash `acmts` default so a partial-knobs literal caller computes
|
||||
// the same trimmed set the cache key claims it did (omitting the field must
|
||||
// not silently disable the floor). Pass minTopScore:0 to disable explicitly.
|
||||
const minTopScore = cfg.minTopScore ?? 0.5;
|
||||
if (top < minTopScore) return noOp(results);
|
||||
|
||||
// Sort a copy descending (A2: don't trust upstream order) and normalize.
|
||||
const sorted = [...scores].sort((a, b) => b - a);
|
||||
const norm = sorted.map((s) => s / top);
|
||||
|
||||
@@ -27,7 +27,7 @@ import { applyAutocut, type AutocutDecision } from './autocut.ts';
|
||||
import { buildRelationalArm } from './relational-recall.ts';
|
||||
import { loadConfigWithEngine } from '../config.ts';
|
||||
import { dedupResults } from './dedup.ts';
|
||||
import { applyReranker } from './rerank.ts';
|
||||
import { applyReranker, rerankerEmitsNormalizedScores } from './rerank.ts';
|
||||
import { autoDetectDetail, classifyQuery, isAmbiguousModalityQuery } from './query-intent.ts';
|
||||
import { isTitlePhraseMatch } from './title-match.ts';
|
||||
import { normalizeAlias } from './alias-normalize.ts';
|
||||
@@ -1572,10 +1572,23 @@ export async function hybridSearch(
|
||||
// never-empty failsafe (1); jumpRatio comes from the resolved mode.
|
||||
let autocutDecision: AutocutDecision | undefined;
|
||||
if (resolvedMode.autocut && offset === 0) {
|
||||
// v0.42.x — minTopScore: weak-top floor. When the top rerank score isn't a
|
||||
// confident anchor, autocut no-ops (fixes the rare-term cross-source
|
||||
// collapse where a weak 0.317 top fabricated a cliff and trimmed 32→1). The
|
||||
// floor is a [0,1] threshold, so it only applies to rerankers that emit
|
||||
// normalized scores; on raw-logit rerankers (llama-server/Qwen3) we pass 0
|
||||
// (disabled) to avoid suppressing legitimate cuts on an unbounded scale.
|
||||
// Gate on the EFFECTIVE reranker model — a per-call SearchOpts.reranker
|
||||
// .model override wins over the bundle default, so reading the bundle alone
|
||||
// would re-open the bug for a per-call raw-logit reranker.
|
||||
const effectiveRerankerModel = rerankerOpts.model ?? resolvedMode.reranker_model;
|
||||
const autocutMinTopScore = rerankerEmitsNormalizedScores(effectiveRerankerModel)
|
||||
? resolvedMode.autocut_min_top_score
|
||||
: 0;
|
||||
const r = applyAutocut(
|
||||
returnPool,
|
||||
(x) => x.rerank_score,
|
||||
{ enabled: true, jumpRatio: resolvedMode.autocut_jump, minKeep: 1 },
|
||||
{ enabled: true, jumpRatio: resolvedMode.autocut_jump, minKeep: 1, minTopScore: autocutMinTopScore },
|
||||
// Preserve alias-hop exact matches: applyAliasHop injects the canonical
|
||||
// page AFTER reranking, so it has no rerank_score. Without this it would
|
||||
// be dropped whenever autocut cuts on the scored set (Codex P1).
|
||||
|
||||
+41
-1
@@ -273,6 +273,16 @@ export interface ModeBundle {
|
||||
relationalRetrieval: boolean;
|
||||
/** v0.43 — max hops for relational traversal. Default 2, hard-capped at 3. */
|
||||
relational_retrieval_depth: number;
|
||||
/**
|
||||
* v0.42.x — autocut weak-top floor: the minimum top cross-encoder rerank
|
||||
* score for the cliff cut to be trusted. autocut normalizes the cliff test by
|
||||
* the top score, so a weak top (e.g. 0.317 on a rare-term cross-source query)
|
||||
* rescales to 1.0 and fabricates a confident cliff — collapsing a rich pool to
|
||||
* 1 (the `--source __all__` collapse). Below this floor autocut no-ops (recall
|
||||
* preserved). Default 0.5 (zerank-2 is bimodal: real ≈0.95+, weak ≈0.3). 0
|
||||
* disables the floor. Override: `search.autocut_min_top_score` config → bundle.
|
||||
*/
|
||||
autocut_min_top_score: number;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -326,6 +336,7 @@ export const MODE_BUNDLES: Readonly<Record<SearchMode, Readonly<ModeBundle>>> =
|
||||
relationalRetrieval: false,
|
||||
relational_retrieval_depth: 2,
|
||||
autocut_jump: 0.2,
|
||||
autocut_min_top_score: 0.5,
|
||||
}),
|
||||
balanced: Object.freeze({
|
||||
cache_enabled: true,
|
||||
@@ -384,6 +395,7 @@ export const MODE_BUNDLES: Readonly<Record<SearchMode, Readonly<ModeBundle>>> =
|
||||
relationalRetrieval: true,
|
||||
relational_retrieval_depth: 2,
|
||||
autocut_jump: 0.2,
|
||||
autocut_min_top_score: 0.5,
|
||||
}),
|
||||
tokenmax: Object.freeze({
|
||||
cache_enabled: true,
|
||||
@@ -435,6 +447,7 @@ export const MODE_BUNDLES: Readonly<Record<SearchMode, Readonly<ModeBundle>>> =
|
||||
relationalRetrieval: true,
|
||||
relational_retrieval_depth: 2,
|
||||
autocut_jump: 0.2,
|
||||
autocut_min_top_score: 0.5,
|
||||
}),
|
||||
});
|
||||
|
||||
@@ -489,6 +502,8 @@ export interface SearchKeyOverrides {
|
||||
relationalRetrieval?: boolean;
|
||||
relational_retrieval_depth?: number;
|
||||
autocut_jump?: number;
|
||||
// v0.42.x — autocut weak-top floor override.
|
||||
autocut_min_top_score?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -538,6 +553,8 @@ export interface SearchPerCallOpts {
|
||||
// v0.43 — relational recall per-call overrides.
|
||||
relationalRetrieval?: boolean;
|
||||
relational_retrieval_depth?: number;
|
||||
// v0.42.x — autocut weak-top floor per-call override.
|
||||
autocut_min_top_score?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -633,6 +650,8 @@ export function resolveSearchMode(input: ResolveSearchModeInput): ResolvedSearch
|
||||
// v0.43 — relational recall resolved via the same pick chain.
|
||||
relationalRetrieval: pick('relationalRetrieval'),
|
||||
relational_retrieval_depth: pick('relational_retrieval_depth'),
|
||||
// v0.42.x — autocut weak-top floor resolved via the same pick chain.
|
||||
autocut_min_top_score: pick('autocut_min_top_score'),
|
||||
resolved_mode,
|
||||
mode_valid: valid,
|
||||
};
|
||||
@@ -756,7 +775,12 @@ export function attributeKnob<K extends keyof ModeBundle>(
|
||||
// slugs written by a process without it, and vice versa. Same one-time
|
||||
// global cold-miss pattern as the bumps above; refills within
|
||||
// cache.ttl_seconds (3600s default).
|
||||
export const KNOBS_HASH_VERSION = 12;
|
||||
//
|
||||
// bump 12→13 (autocut weak-top floor): adds `acmts` (autocut_min_top_score).
|
||||
// The floor changes WHETHER autocut cuts at all — a write made with one floor
|
||||
// must NOT be served to a lookup at a different floor (the trimmed-vs-full set
|
||||
// differs). Same one-time global cold-miss pattern; fills within cache.ttl.
|
||||
export const KNOBS_HASH_VERSION = 13;
|
||||
|
||||
/**
|
||||
* v0.36 (D8 / CDX-2) — second-arg context for the cache key. The
|
||||
@@ -888,6 +912,13 @@ export function knobsHash(
|
||||
// across processes. Sorted copy so ['a/','b/'] and ['b/','a/'] hash
|
||||
// identically; undefined falls back to 'none' for legacy callers.
|
||||
`hx=${ctx?.hardExcludes ? [...ctx.hardExcludes].sort().join(',') : 'none'}`,
|
||||
// v=13 addition (append-only) — autocut weak-top floor shifts whether
|
||||
// autocut cuts at all, so an autocut-cut write must not be served to a
|
||||
// different-floor lookup. `?? 0.5` mirrors the module default for
|
||||
// partial-knobs callers. 4 decimals (vs acj's 2): the floor is compared
|
||||
// directly against raw rerank scores, so nearby config values (0.501 vs
|
||||
// 0.504) can flip trim-vs-no-op and must not collide on the cache key.
|
||||
`acmts=${(knobs.autocut_min_top_score ?? 0.5).toFixed(4)}`,
|
||||
];
|
||||
const h = createHash('sha256');
|
||||
h.update(parts.join('|'));
|
||||
@@ -1054,6 +1085,13 @@ export function loadOverridesFromConfig(
|
||||
const n = parseFloat(acj);
|
||||
if (Number.isFinite(n) && n > 0 && n <= 1) out.autocut_jump = n;
|
||||
}
|
||||
// v0.42.x — autocut weak-top floor. [0, 1]: 0 disables, 1 pins at ceiling;
|
||||
// out-of-range falls through to the bundle.
|
||||
const acmts = get('search.autocut_min_top_score');
|
||||
if (acmts !== undefined) {
|
||||
const n = parseFloat(acmts);
|
||||
if (Number.isFinite(n) && n >= 0 && n <= 1) out.autocut_min_top_score = n;
|
||||
}
|
||||
|
||||
// v0.43 — relational recall arm.
|
||||
const rel = get('search.relational_retrieval');
|
||||
@@ -1108,6 +1146,8 @@ export const SEARCH_MODE_CONFIG_KEYS: ReadonlyArray<string> = Object.freeze([
|
||||
'search.relational_retrieval',
|
||||
'search.relational_retrieval_depth',
|
||||
'search.autocut_jump',
|
||||
// v0.42.x autocut weak-top floor
|
||||
'search.autocut_min_top_score',
|
||||
]);
|
||||
|
||||
/**
|
||||
|
||||
@@ -39,6 +39,30 @@ export interface RerankerOpts {
|
||||
rerankerFn?: (input: RerankInput) => Promise<RerankResult[]>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Does this reranker emit relevance scores normalized to roughly [0, 1]?
|
||||
*
|
||||
* `rerank_score` is taken verbatim from the provider response (gateway.ts —
|
||||
* no sigmoid, no normalization). Cloud rerankers (ZeroEntropy zerank-*, Cohere,
|
||||
* Voyage) return calibrated [0, 1] relevance. But the self-hosted
|
||||
* `llama-server-reranker` path (Qwen3-Reranker GGUF via llama.cpp `--reranking`)
|
||||
* returns the RAW cross-encoder logit — unbounded, frequently negative for
|
||||
* non-matches and well above 1 for strong matches.
|
||||
*
|
||||
* This matters for autocut's weak-top floor (autocut.ts minTopScore): the floor
|
||||
* is a [0, 1] threshold and is meaningless on a raw-logit scale, where it would
|
||||
* either suppress legitimate cuts (a confident logit that happens to land in
|
||||
* (0, floor)) or no-op entirely (logits ≫ 1). Callers gate the floor on this.
|
||||
*
|
||||
* Conservative: returns false ONLY for the known raw-logit local rerankers, so
|
||||
* an unrecognized cloud model still gets the (correct) normalized treatment.
|
||||
*/
|
||||
export function rerankerEmitsNormalizedScores(model?: string): boolean {
|
||||
if (!model) return true; // default reranker is ZE zerank-2 ([0,1])
|
||||
const m = model.toLowerCase();
|
||||
return !(m.includes('llama-server-reranker') || m.includes('qwen3-reranker'));
|
||||
}
|
||||
|
||||
/** SHA-256 prefix (8 chars) of the query text for privacy-preserving audit. */
|
||||
function hashQuery(query: string): string {
|
||||
return createHash('sha256').update(query, 'utf8').digest('hex').slice(0, 8);
|
||||
|
||||
@@ -136,7 +136,7 @@ describe('D2 — knobsHash differs across cross-modal knob values', () => {
|
||||
return resolveSearchMode({ mode: 'balanced' });
|
||||
}
|
||||
|
||||
test('KNOBS_HASH_VERSION is 12 (cross-modal still appended; 11→12 hard-exclude fold #2825)', () => {
|
||||
test('KNOBS_HASH_VERSION is 13 (cross-modal still appended; 12→13 autocut weak-top floor)', () => {
|
||||
// v0.35 ladder: 1→2 reranker, 2→3 floor_ratio. v0.36 piggybacks on v=3
|
||||
// with 7 cross-modal knobs + column/provider context. v0.40.4 (salem) +
|
||||
// v0.39 T21 (master) bump to v=4 for graph_signals + schema-pack fields.
|
||||
@@ -146,7 +146,8 @@ describe('D2 — knobsHash differs across cross-modal knob values', () => {
|
||||
// v0.43: 9→10 relational recall arm. #1400: 10→11 query-side input_type
|
||||
// finally reaches asymmetric providers — pre-fix rows were keyed on
|
||||
// document-side query vectors. #2825: 11→12 hard-exclude fold (hx=).
|
||||
expect(KNOBS_HASH_VERSION).toBe(12);
|
||||
// 12→13: autocut weak-top floor (acmts=).
|
||||
expect(KNOBS_HASH_VERSION).toBe(13);
|
||||
});
|
||||
|
||||
test('flipping unified_multimodal changes the hash', () => {
|
||||
|
||||
@@ -89,7 +89,7 @@ describe('alias_resolved boost stage', () => {
|
||||
});
|
||||
|
||||
describe('KNOBS_HASH_VERSION', () => {
|
||||
it('is 12 (11→12 hard-exclude fold invalidates rows written under a different exclude policy, #2825)', () => {
|
||||
expect(KNOBS_HASH_VERSION).toBe(12);
|
||||
it('is 13 (12→13 autocut weak-top floor acmts= added; 11→12 was #2825 hard-exclude fold)', () => {
|
||||
expect(KNOBS_HASH_VERSION).toBe(13);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -80,6 +80,7 @@ describe('SEARCH_MODES + MODE_BUNDLES canonical shape', () => {
|
||||
// v0.43 — relational recall OFF for conservative.
|
||||
relationalRetrieval: false,
|
||||
relational_retrieval_depth: 2,
|
||||
autocut_min_top_score: 0.5,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -112,6 +113,7 @@ describe('SEARCH_MODES + MODE_BUNDLES canonical shape', () => {
|
||||
// v0.43 — relational recall ON for balanced.
|
||||
relationalRetrieval: true,
|
||||
relational_retrieval_depth: 2,
|
||||
autocut_min_top_score: 0.5,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -142,6 +144,7 @@ describe('SEARCH_MODES + MODE_BUNDLES canonical shape', () => {
|
||||
// v0.43 — relational recall ON for tokenmax.
|
||||
relationalRetrieval: true,
|
||||
relational_retrieval_depth: 2,
|
||||
autocut_min_top_score: 0.5,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -409,8 +412,9 @@ describe('knobsHash determinism + cross-mode separation (CDX-4)', () => {
|
||||
// must not be served to post-fix lookups.
|
||||
// #2825: bumped 11→12 to fold the resolved hard-exclude prefix list
|
||||
// (hx=) — cached rows leaked GBRAIN_SEARCH_EXCLUDE'd slugs across
|
||||
// processes.
|
||||
expect(KNOBS_HASH_VERSION).toBe(12);
|
||||
// processes. 12→13: autocut weak-top floor (acmts=) — the floor shifts
|
||||
// whether autocut cuts at all.
|
||||
expect(KNOBS_HASH_VERSION).toBe(13);
|
||||
});
|
||||
|
||||
test('T1 (codex): floor_ratio set vs unset produces DIFFERENT hashes (cache contamination prevention)', () => {
|
||||
@@ -575,8 +579,8 @@ describe('v0.40.4 — graph_signals knob', () => {
|
||||
});
|
||||
|
||||
describe('v0.42.3.0 — autocut knobs', () => {
|
||||
test('KNOBS_HASH_VERSION is 12 (11→12 hard-exclude fold, #2825)', () => {
|
||||
expect(KNOBS_HASH_VERSION).toBe(12);
|
||||
test('KNOBS_HASH_VERSION is 13 (12→13 autocut weak-top floor; 11→12 was #2825 hard-excludes)', () => {
|
||||
expect(KNOBS_HASH_VERSION).toBe(13);
|
||||
});
|
||||
|
||||
test('bundle defaults: conservative off, balanced/tokenmax on @0.20', () => {
|
||||
@@ -585,6 +589,8 @@ describe('v0.42.3.0 — autocut knobs', () => {
|
||||
expect(MODE_BUNDLES.tokenmax.autocut).toBe(true);
|
||||
for (const m of ['conservative', 'balanced', 'tokenmax'] as const) {
|
||||
expect(MODE_BUNDLES[m].autocut_jump).toBe(0.2);
|
||||
// v0.42.x — weak-top floor default 0.5 across all bundles.
|
||||
expect(MODE_BUNDLES[m].autocut_min_top_score).toBe(0.5);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -614,9 +620,37 @@ describe('v0.42.3.0 — autocut knobs', () => {
|
||||
expect(ov.autocut_jump).toBe(0.35);
|
||||
});
|
||||
|
||||
test('resolveSearchMode threads autocut_min_top_score: per-call > config > bundle', () => {
|
||||
expect(resolveSearchMode({ mode: 'balanced' }).autocut_min_top_score).toBe(0.5);
|
||||
expect(
|
||||
resolveSearchMode({ mode: 'balanced', overrides: { autocut_min_top_score: 0.7 } }).autocut_min_top_score,
|
||||
).toBe(0.7);
|
||||
expect(
|
||||
resolveSearchMode({
|
||||
mode: 'balanced',
|
||||
overrides: { autocut_min_top_score: 0.7 },
|
||||
perCall: { autocut_min_top_score: 0.3 },
|
||||
}).autocut_min_top_score,
|
||||
).toBe(0.3);
|
||||
});
|
||||
|
||||
test('loadOverridesFromConfig reads search.autocut_min_top_score (clamped [0,1])', () => {
|
||||
expect(loadOverridesFromConfig({ 'search.autocut_min_top_score': '0.7' }).autocut_min_top_score).toBe(0.7);
|
||||
expect(loadOverridesFromConfig({ 'search.autocut_min_top_score': '0' }).autocut_min_top_score).toBe(0);
|
||||
// out of range → ignored (undefined, falls through to bundle)
|
||||
expect(loadOverridesFromConfig({ 'search.autocut_min_top_score': '1.5' }).autocut_min_top_score).toBeUndefined();
|
||||
});
|
||||
|
||||
test('SEARCH_MODE_CONFIG_KEYS includes the autocut keys', () => {
|
||||
expect(SEARCH_MODE_CONFIG_KEYS).toContain('search.autocut');
|
||||
expect(SEARCH_MODE_CONFIG_KEYS).toContain('search.autocut_jump');
|
||||
expect(SEARCH_MODE_CONFIG_KEYS).toContain('search.autocut_min_top_score');
|
||||
});
|
||||
|
||||
test('knobsHash includes acmts= — different weak-top floors differ', () => {
|
||||
const a = knobsHash(resolveSearchMode({ mode: 'balanced' })); // floor 0.5
|
||||
const b = knobsHash(resolveSearchMode({ mode: 'balanced', perCall: { autocut_min_top_score: 0.3 } }));
|
||||
expect(a).not.toBe(b);
|
||||
});
|
||||
|
||||
test('knobsHash includes ac= / acj= — autocut-on vs off differ', () => {
|
||||
|
||||
@@ -156,6 +156,53 @@ describe('autocut — ceiling override', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('autocut — weak-top floor (cross-source collapse regression)', () => {
|
||||
// The `--source __all__` collapse: on a rare-term cross-source query the top
|
||||
// reranker score is WEAK (~0.317) yet, because autocut normalizes by the top,
|
||||
// the normal decay below it looked like a confident cliff and trimmed a
|
||||
// 32-result pool to 1. The weak-top floor (bundle default 0.5) no-ops autocut
|
||||
// when the top isn't a confident anchor. Pinned end-to-end (the pure-fn tests
|
||||
// can't prove it flows through resolveSearchMode → applyAutocut).
|
||||
test('cliff after rank 1 but WEAK top (0.317) → no trim, recall preserved', async () => {
|
||||
const baseline = await hybridSearch(engine, 'alpha keyword', { limit: 10 });
|
||||
// Exact live-brain shape that collapsed: 0.317 / 0.197 / 0.131 / 0.118.
|
||||
const out = await hybridSearch(engine, 'alpha keyword', {
|
||||
limit: 10,
|
||||
reranker: rerankerOpts([0.317, 0.197, 0.131, 0.118, 0.1]),
|
||||
});
|
||||
expect(out.length).toBe(baseline.length);
|
||||
expect(out.length).toBeGreaterThanOrEqual(3);
|
||||
});
|
||||
|
||||
test('same cliff shape but STRONG top (0.95) → trims to the confident answer', async () => {
|
||||
// top 0.95 >= floor; normalized rank1→rank2 gap (1.0→0.62 = 0.38) cuts.
|
||||
const out = await hybridSearch(engine, 'alpha keyword', {
|
||||
limit: 10,
|
||||
reranker: rerankerOpts([0.95, 0.59, 0.39, 0.35, 0.3]),
|
||||
});
|
||||
expect(out.length).toBe(1);
|
||||
});
|
||||
|
||||
test('per-call RAW-LOGIT reranker model → floor disabled, weak-top cliff still trims', async () => {
|
||||
// The floor is gated on the EFFECTIVE reranker model. A per-call
|
||||
// SearchOpts.reranker.model override to a raw-logit local reranker
|
||||
// (llama-server/Qwen3) must disable the [0,1] floor — otherwise the weak
|
||||
// 0.317 top would be wrongly read as "not confident" on an unbounded scale.
|
||||
// With the floor off, the cliff (0.317→0.197) is trusted again → trims.
|
||||
const out = await hybridSearch(engine, 'alpha keyword', {
|
||||
limit: 10,
|
||||
reranker: {
|
||||
enabled: true,
|
||||
topNIn: 30,
|
||||
topNOut: null,
|
||||
model: 'llama-server-reranker:Qwen3-Reranker-4B',
|
||||
rerankerFn: rerankerWithScores([0.317, 0.197, 0.131, 0.118, 0.1]),
|
||||
},
|
||||
});
|
||||
expect(out.length).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('autocut — composes with adaptive-return (never-empty holds)', () => {
|
||||
test('adaptive-return + autocut both on → non-empty, bounded', async () => {
|
||||
const out = await hybridSearch(engine, 'alpha keyword', {
|
||||
|
||||
@@ -18,7 +18,7 @@ import {
|
||||
// items survived the cut.
|
||||
type R = { id: string; rs?: number };
|
||||
const scoreOf = (r: R) => r.rs;
|
||||
const ON: AutocutConfig = { enabled: true, jumpRatio: 0.2, minKeep: 1 };
|
||||
const ON: AutocutConfig = { enabled: true, jumpRatio: 0.2, minKeep: 1, minTopScore: 0.5 };
|
||||
|
||||
function mk(scores: Array<number | undefined>): R[] {
|
||||
return scores.map((rs, i) => ({ id: `r${i}`, rs }));
|
||||
@@ -33,6 +33,12 @@ describe('DEFAULT_AUTOCUT', () => {
|
||||
test('is frozen', () => {
|
||||
expect(Object.isFrozen(DEFAULT_AUTOCUT)).toBe(true);
|
||||
});
|
||||
test('module default carries a weak-top floor (minTopScore 0.5)', () => {
|
||||
// The reranker (zerank-2) is bimodal: real matches score ≈0.95+, weak
|
||||
// matches ≈0.3. Below this floor the top is not a confident anchor, so
|
||||
// normalizing by it manufactures a false cliff (see weak-top-floor block).
|
||||
expect(DEFAULT_AUTOCUT.minTopScore).toBe(0.5);
|
||||
});
|
||||
});
|
||||
|
||||
describe('applyAutocut — cuts on a real cliff', () => {
|
||||
@@ -136,6 +142,56 @@ describe('applyAutocut — no-op guards', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('applyAutocut — weak-top floor (cross-source collapse fix)', () => {
|
||||
// Root cause of the `--source __all__` collapse: applyAutocut normalizes the
|
||||
// cliff test by the top score, so a WEAK top (e.g. 0.317 on a rare-term
|
||||
// cross-source query) gets rescaled to 1.0 and a normal decay below it looks
|
||||
// like a confident cliff → the 32-result pool collapses to 1. Calibration on
|
||||
// the live brain: every legitimate single-answer collapse has top_rerank≥0.95;
|
||||
// the only bug case had top_rerank=0.317. minTopScore=0.5 sits in the empty
|
||||
// middle of the reranker's bimodal distribution.
|
||||
|
||||
test('weak top below floor → NO cut even when a cliff exists (recall preserved)', () => {
|
||||
// The exact live-brain shape that collapsed: 0.317 / 0.197 / 0.131 / 0.118.
|
||||
// Normalized, rank1→rank2 gap is 0.38 (clears jumpRatio 0.20) — but the top
|
||||
// is a weak 0.317, so the cliff is untrustworthy. Keep all 4.
|
||||
const r = applyAutocut(mk([0.317, 0.197, 0.131, 0.118]), scoreOf, ON);
|
||||
expect(r.kept.length).toBe(4);
|
||||
expect(r.decision.applied).toBe(false);
|
||||
expect(r.decision.signal).toBe('none');
|
||||
});
|
||||
|
||||
test('strong top above floor → still cuts on a real cliff (precision preserved)', () => {
|
||||
// The legitimate single-answer case: top 0.984 is a confident anchor and the
|
||||
// cliff after rank 1 is real. The floor must NOT suppress this.
|
||||
const r = applyAutocut(mk([0.984, 0.1, 0.08, 0.05]), scoreOf, ON);
|
||||
expect(r.kept.map((x) => x.id)).toEqual(['r0']);
|
||||
expect(r.decision.applied).toBe(true);
|
||||
});
|
||||
|
||||
test('top exactly at the floor → treated as confident (>= floor cuts)', () => {
|
||||
const r = applyAutocut(mk([0.5, 0.1, 0.05]), scoreOf, ON);
|
||||
expect(r.kept.map((x) => x.id)).toEqual(['r0']);
|
||||
expect(r.decision.applied).toBe(true);
|
||||
});
|
||||
|
||||
test('minTopScore 0 disables the floor (explicit pre-fix behavior)', () => {
|
||||
const r = applyAutocut(mk([0.317, 0.197, 0.131, 0.118]), scoreOf, { ...ON, minTopScore: 0 });
|
||||
expect(r.kept.length).toBe(1);
|
||||
expect(r.decision.applied).toBe(true);
|
||||
});
|
||||
|
||||
test('partial-knobs caller omitting minTopScore defaults to the 0.5 floor', () => {
|
||||
// A literal that predates the field (or a JS caller) must compute the SAME
|
||||
// trimmed set the knobsHash `acmts` default (0.5) claims — omitting the
|
||||
// field must NOT silently disable the floor (compute/cache agreement).
|
||||
const partial = { enabled: true, jumpRatio: 0.2, minKeep: 1 } as AutocutConfig;
|
||||
const r = applyAutocut(mk([0.317, 0.197, 0.131, 0.118]), scoreOf, partial);
|
||||
expect(r.kept.length).toBe(4); // weak top 0.317 < default 0.5 → no cut
|
||||
expect(r.decision.applied).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('applyAutocut — failsafe', () => {
|
||||
test('never returns empty when input is non-empty', () => {
|
||||
const r = applyAutocut(mk([0.9, 0.01]), scoreOf, ON);
|
||||
@@ -208,6 +264,15 @@ describe('autocutFromConfig', () => {
|
||||
expect(autocutFromConfig({ search: { autocut_jump: 5 } }).jumpRatio).toBeUndefined();
|
||||
expect(autocutFromConfig({ search: { autocut_jump: 0 } }).jumpRatio).toBeUndefined();
|
||||
});
|
||||
test('reads search.autocut_min_top_score (clamped to [0, 1])', () => {
|
||||
expect(autocutFromConfig({ search: { autocut_min_top_score: 0.7 } }).minTopScore).toBe(0.7);
|
||||
// 0 is valid (disables the floor); 1 is valid (floor at the ceiling).
|
||||
expect(autocutFromConfig({ search: { autocut_min_top_score: 0 } }).minTopScore).toBe(0);
|
||||
expect(autocutFromConfig({ search: { autocut_min_top_score: 1 } }).minTopScore).toBe(1);
|
||||
// Out of range → ignored (fall through to bundle/default).
|
||||
expect(autocutFromConfig({ search: { autocut_min_top_score: -0.1 } }).minTopScore).toBeUndefined();
|
||||
expect(autocutFromConfig({ search: { autocut_min_top_score: 1.5 } }).minTopScore).toBeUndefined();
|
||||
});
|
||||
test('empty / missing config → empty partial', () => {
|
||||
expect(autocutFromConfig(null)).toEqual({});
|
||||
expect(autocutFromConfig({})).toEqual({});
|
||||
@@ -229,4 +294,9 @@ describe('resolveAutocut — precedence ladder', () => {
|
||||
expect(cfg.jumpRatio).toBe(0.5);
|
||||
expect(cfg.enabled).toBe(false); // inherited from config (partial didn't set it)
|
||||
});
|
||||
test('minTopScore resolves default → config → per-call', () => {
|
||||
expect(resolveAutocut(undefined, undefined).minTopScore).toBe(0.5); // module default
|
||||
expect(resolveAutocut(undefined, { minTopScore: 0.7 }).minTopScore).toBe(0.7); // config
|
||||
expect(resolveAutocut({ minTopScore: 0.3 }, { minTopScore: 0.7 }).minTopScore).toBe(0.3); // per-call wins
|
||||
});
|
||||
});
|
||||
|
||||
@@ -44,7 +44,7 @@ function baseKnobs(): ResolvedSearchKnobs {
|
||||
}
|
||||
|
||||
describe('KNOBS_HASH_VERSION + version invariants', () => {
|
||||
test('version is 12 (…; 9→10 relational recall; 10→11 asymmetric input_type #1400; 11→12 hard-excludes #2825)', () => {
|
||||
test('version is 13 (…; 10→11 asymmetric input_type #1400; 11→12 hard-excludes #2825; 12→13 autocut weak-top floor)', () => {
|
||||
// v0.35.0.0: 1→2 to fold reranker fields. v0.35.6.0: 2→3 to fold
|
||||
// floor_ratio. v0.36 wave: piggybacks on v=3 with 7 cross-modal knobs
|
||||
// (D2) PLUS column + provider context (D8/CDX-2 cross-column isolation).
|
||||
@@ -64,7 +64,9 @@ describe('KNOBS_HASH_VERSION + version invariants', () => {
|
||||
// pre-fix document-side query vectors must not be served.
|
||||
// #2825: 11→12 to fold the resolved hard-exclude prefix list (hx=) —
|
||||
// cached rows leaked GBRAIN_SEARCH_EXCLUDE'd slugs across processes.
|
||||
expect(KNOBS_HASH_VERSION).toBe(12);
|
||||
// 12→13: autocut weak-top floor (acmts=) — the floor shifts whether
|
||||
// autocut cuts at all, so a different-floor write must not be served.
|
||||
expect(KNOBS_HASH_VERSION).toBe(13);
|
||||
});
|
||||
|
||||
test('hash is 16 hex chars regardless of reranker config', () => {
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { applyReranker, type RerankerOpts } from '../../src/core/search/rerank.ts';
|
||||
import { applyReranker, rerankerEmitsNormalizedScores, type RerankerOpts } from '../../src/core/search/rerank.ts';
|
||||
import { RerankError, type RerankResult } from '../../src/core/ai/gateway.ts';
|
||||
import type { SearchResult } from '../../src/core/types.ts';
|
||||
|
||||
@@ -215,3 +215,22 @@ describe('applyReranker — pass-through cases', () => {
|
||||
expect(called).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('rerankerEmitsNormalizedScores (autocut weak-top-floor gate)', () => {
|
||||
test('cloud rerankers emit [0,1] scores → true', () => {
|
||||
expect(rerankerEmitsNormalizedScores('zeroentropyai:zerank-2')).toBe(true);
|
||||
expect(rerankerEmitsNormalizedScores('zeroentropyai:zerank-1-small')).toBe(true);
|
||||
expect(rerankerEmitsNormalizedScores('cohere:rerank-v3.5')).toBe(true);
|
||||
expect(rerankerEmitsNormalizedScores('voyage:rerank-2')).toBe(true);
|
||||
});
|
||||
test('undefined (default ZE reranker) → true', () => {
|
||||
expect(rerankerEmitsNormalizedScores(undefined)).toBe(true);
|
||||
expect(rerankerEmitsNormalizedScores('')).toBe(true);
|
||||
});
|
||||
test('raw-logit local rerankers (llama-server / Qwen3) → false', () => {
|
||||
// These emit unbounded cross-encoder logits, so a [0,1] floor is meaningless.
|
||||
expect(rerankerEmitsNormalizedScores('llama-server-reranker:Qwen3-Reranker-4B')).toBe(false);
|
||||
expect(rerankerEmitsNormalizedScores('Qwen3-Reranker-0.6B')).toBe(false);
|
||||
expect(rerankerEmitsNormalizedScores('LLAMA-SERVER-RERANKER:foo')).toBe(false); // case-insensitive
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user