From 9fca071458a142ecbbe573a1c6918b3f1388c003 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Thu, 23 Jul 2026 11:00:19 -0700 Subject: [PATCH] fix(search): fold resolved source-boost map into query-cache knobs hash (takeover of #3213) Boosts reorder results at query-build time only, so a ranking-policy change (GBRAIN_SOURCE_BOOST tune or new defaults) kept serving rows ranked under the old policy for up to cache.ttl_seconds. Folds resolveBoostMap() into the cache key (KNOBS_HASH_VERSION 12 -> 13, canonicalized sb= part, same pattern as #2825's hx=). Repair over #3213: the PR left two KNOBS_HASH_VERSION pins at 12 (test/cross-modal-phase1.test.ts, test/search-alias-resolved-boost.test.ts), which would have failed CI. Both updated to 13 with the bump rationale. Co-authored-by: SVZTeam Co-Authored-By: Claude Fable 5 --- src/core/search/hybrid.ts | 7 ++++- src/core/search/mode.ts | 34 ++++++++++++++++++++- test/cross-modal-phase1.test.ts | 6 ++-- test/search-alias-resolved-boost.test.ts | 4 +-- test/search-mode.test.ts | 9 ++++-- test/search/knobs-hash-reranker.test.ts | 39 ++++++++++++++++++++++-- 6 files changed, 87 insertions(+), 12 deletions(-) diff --git a/src/core/search/hybrid.ts b/src/core/search/hybrid.ts index de0a95eb4..8aadc3b83 100644 --- a/src/core/search/hybrid.ts +++ b/src/core/search/hybrid.ts @@ -15,7 +15,7 @@ import type { SearchResult, SearchOpts, HybridSearchMeta } from '../types.ts'; import { embed, embedQuery } from '../embedding.ts'; import { registerBackgroundWorkDrainer } from '../background-work.ts'; import { resolveEmbeddingColumn, isCacheSafe } from './embedding-column.ts'; -import { resolveHardExcludes } from './source-boost.ts'; +import { resolveHardExcludes, resolveBoostMap } from './source-boost.ts'; import { resolveAdaptiveReturn, applyAdaptiveReturn, @@ -1702,6 +1702,11 @@ export async function hybridSearchCached( // resolves) into the cache key so a row written under one exclude // policy can't be served to a lookup under another. hardExcludes: resolveHardExcludes(opts?.exclude_slug_prefixes, opts?.include_slug_prefixes), + // v=13 — fold the resolved source-boost map (defaults merged with + // GBRAIN_SOURCE_BOOST) into the cache key. Boosts reorder results at + // query-build time only, so without this a ranking-policy change kept + // serving rows ranked under the old policy for up to cache.ttl_seconds. + sourceBoostMap: resolveBoostMap(), }); // Cache decision: opts.useCache (explicit) wins over global config; global diff --git a/src/core/search/mode.ts b/src/core/search/mode.ts index cb5a14782..3d397db70 100644 --- a/src/core/search/mode.ts +++ b/src/core/search/mode.ts @@ -756,7 +756,16 @@ export function attributeKnob( // slugs written by a process without it, and vice versa. Same one-time // global cold-miss pattern as the bumps above; refills within // cache.ttl_seconds (3600s default). -export const KNOBS_HASH_VERSION = 12; +// +// bump 12→13: the resolved source-boost map (DEFAULT_SOURCE_BOOSTS merged +// with GBRAIN_SOURCE_BOOST) folds into the key via ctx.sourceBoostMap. Same +// bug class as #2825's hard-excludes: boosts only applied at DB-query build +// time (cache miss), so after a ranking-policy change (env tune or upgrade +// that ships new defaults) lookups kept being served rows ranked under the +// OLD policy for up to cache.ttl_seconds — making boost tuning appear +// nondeterministic. Same one-time global cold-miss pattern; refills within +// cache.ttl_seconds (3600s default). +export const KNOBS_HASH_VERSION = 13; /** * v0.36 (D8 / CDX-2) — second-arg context for the cache key. The @@ -795,6 +804,17 @@ export interface KnobsHashContext { * 'none' for legacy callers that don't thread excludes. */ hardExcludes?: string[]; + /** + * v=13: the RESOLVED effective source-boost map — the same value + * resolveBoostMap() produces at query-build time (DEFAULT_SOURCE_BOOSTS + * merged with GBRAIN_SOURCE_BOOST). Ranking priors change result ORDER, + * not membership, so they are invisible to the hard-exclude fold above — + * yet a cache row ranked under an old boost policy is exactly as stale as + * one filtered under an old exclude policy. Canonicalized (entries sorted + * by prefix) so object-key insertion order is irrelevant. Undefined falls + * back to the literal 'default' for legacy callers. + */ + sourceBoostMap?: Record; } export function knobsHash( @@ -888,6 +908,18 @@ export function knobsHash( // across processes. Sorted copy so ['a/','b/'] and ['b/','a/'] hash // identically; undefined falls back to 'none' for legacy callers. `hx=${ctx?.hardExcludes ? [...ctx.hardExcludes].sort().join(',') : 'none'}`, + // v=13 addition (append-only): resolved source-boost map. Boost changes + // reorder results, so a row ranked under one policy must not be served + // to a lookup under another. Entries sorted by prefix so map insertion + // order is irrelevant; undefined falls back to 'default'. + `sb=${ + ctx?.sourceBoostMap + ? Object.entries(ctx.sourceBoostMap) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([prefix, factor]) => `${prefix}:${factor}`) + .join(',') + : 'default' + }`, ]; const h = createHash('sha256'); h.update(parts.join('|')); diff --git a/test/cross-modal-phase1.test.ts b/test/cross-modal-phase1.test.ts index 19689a324..398ee9f0a 100644 --- a/test/cross-modal-phase1.test.ts +++ b/test/cross-modal-phase1.test.ts @@ -136,7 +136,7 @@ describe('D2 — knobsHash differs across cross-modal knob values', () => { return resolveSearchMode({ mode: 'balanced' }); } - test('KNOBS_HASH_VERSION is 12 (cross-modal still appended; 11→12 hard-exclude fold #2825)', () => { + test('KNOBS_HASH_VERSION is 13 (cross-modal still appended; 12→13 source-boost map fold)', () => { // v0.35 ladder: 1→2 reranker, 2→3 floor_ratio. v0.36 piggybacks on v=3 // with 7 cross-modal knobs + column/provider context. v0.40.4 (salem) + // v0.39 T21 (master) bump to v=4 for graph_signals + schema-pack fields. @@ -146,7 +146,9 @@ describe('D2 — knobsHash differs across cross-modal knob values', () => { // v0.43: 9→10 relational recall arm. #1400: 10→11 query-side input_type // finally reaches asymmetric providers — pre-fix rows were keyed on // document-side query vectors. #2825: 11→12 hard-exclude fold (hx=). - expect(KNOBS_HASH_VERSION).toBe(12); + // 12→13: source-boost map fold (sb=) — a ranking-policy change must not + // be served rows ranked under the previous policy. + expect(KNOBS_HASH_VERSION).toBe(13); }); test('flipping unified_multimodal changes the hash', () => { diff --git a/test/search-alias-resolved-boost.test.ts b/test/search-alias-resolved-boost.test.ts index 224d2d988..89d40feb6 100644 --- a/test/search-alias-resolved-boost.test.ts +++ b/test/search-alias-resolved-boost.test.ts @@ -89,7 +89,7 @@ describe('alias_resolved boost stage', () => { }); describe('KNOBS_HASH_VERSION', () => { - it('is 12 (11→12 hard-exclude fold invalidates rows written under a different exclude policy, #2825)', () => { - expect(KNOBS_HASH_VERSION).toBe(12); + it('is 13 (12→13 source-boost map fold invalidates rows ranked under a different boost policy)', () => { + expect(KNOBS_HASH_VERSION).toBe(13); }); }); diff --git a/test/search-mode.test.ts b/test/search-mode.test.ts index 223d112d9..aee7260c2 100644 --- a/test/search-mode.test.ts +++ b/test/search-mode.test.ts @@ -410,7 +410,10 @@ describe('knobsHash determinism + cross-mode separation (CDX-4)', () => { // #2825: bumped 11→12 to fold the resolved hard-exclude prefix list // (hx=) — cached rows leaked GBRAIN_SEARCH_EXCLUDE'd slugs across // processes. - expect(KNOBS_HASH_VERSION).toBe(12); + // bumped 12→13 to fold the resolved source-boost map (sb=) — a + // ranking-policy change (GBRAIN_SOURCE_BOOST tune or new defaults) + // must not be served rows ranked under the previous policy. + expect(KNOBS_HASH_VERSION).toBe(13); }); test('T1 (codex): floor_ratio set vs unset produces DIFFERENT hashes (cache contamination prevention)', () => { @@ -575,8 +578,8 @@ describe('v0.40.4 — graph_signals knob', () => { }); describe('v0.42.3.0 — autocut knobs', () => { - test('KNOBS_HASH_VERSION is 12 (11→12 hard-exclude fold, #2825)', () => { - expect(KNOBS_HASH_VERSION).toBe(12); + test('KNOBS_HASH_VERSION is 13 (12→13 source-boost map fold)', () => { + expect(KNOBS_HASH_VERSION).toBe(13); }); test('bundle defaults: conservative off, balanced/tokenmax on @0.20', () => { diff --git a/test/search/knobs-hash-reranker.test.ts b/test/search/knobs-hash-reranker.test.ts index 9f73ac394..2414130a1 100644 --- a/test/search/knobs-hash-reranker.test.ts +++ b/test/search/knobs-hash-reranker.test.ts @@ -27,7 +27,7 @@ import { MODE_BUNDLES, type ResolvedSearchKnobs, } from '../../src/core/search/mode.ts'; -import { resolveHardExcludes } from '../../src/core/search/source-boost.ts'; +import { resolveHardExcludes, resolveBoostMap } from '../../src/core/search/source-boost.ts'; /** Build a baseline resolved knob set with all reranker fields filled. */ function baseKnobs(): ResolvedSearchKnobs { @@ -44,7 +44,7 @@ function baseKnobs(): ResolvedSearchKnobs { } describe('KNOBS_HASH_VERSION + version invariants', () => { - test('version is 12 (…; 9→10 relational recall; 10→11 asymmetric input_type #1400; 11→12 hard-excludes #2825)', () => { + test('version is 13 (…; 10→11 asymmetric input_type #1400; 11→12 hard-excludes #2825; 12→13 source-boost map)', () => { // v0.35.0.0: 1→2 to fold reranker fields. v0.35.6.0: 2→3 to fold // floor_ratio. v0.36 wave: piggybacks on v=3 with 7 cross-modal knobs // (D2) PLUS column + provider context (D8/CDX-2 cross-column isolation). @@ -64,7 +64,9 @@ describe('KNOBS_HASH_VERSION + version invariants', () => { // pre-fix document-side query vectors must not be served. // #2825: 11→12 to fold the resolved hard-exclude prefix list (hx=) — // cached rows leaked GBRAIN_SEARCH_EXCLUDE'd slugs across processes. - expect(KNOBS_HASH_VERSION).toBe(12); + // 12→13 to fold the resolved source-boost map (sb=) — a ranking-policy + // change must not be served rows ranked under the previous policy. + expect(KNOBS_HASH_VERSION).toBe(13); }); test('hash is 16 hex chars regardless of reranker config', () => { @@ -244,3 +246,34 @@ describe('v=12 hard-exclude participation (#2825)', () => { ); }); }); + +describe('v=13 source-boost map participation', () => { + test('different boost maps → different hashes', () => { + const k = baseKnobs(); + const defaults = knobsHash(k, { sourceBoostMap: resolveBoostMap(undefined) }); + const tuned = knobsHash(k, { sourceBoostMap: resolveBoostMap('originals/:1.8,daily/:0.4') }); + expect(defaults).not.toBe(tuned); + }); + + test('same map with different key insertion order → SAME hash (canonicalization)', () => { + const k = baseKnobs(); + const a = knobsHash(k, { sourceBoostMap: { 'a/': 1.2, 'b/': 0.7 } }); + const b = knobsHash(k, { sourceBoostMap: { 'b/': 0.7, 'a/': 1.2 } }); + expect(a).toBe(b); + }); + + test('factor change on a single prefix changes the hash', () => { + const k = baseKnobs(); + const a = knobsHash(k, { sourceBoostMap: { 'skills/': 1.2 } }); + const b = knobsHash(k, { sourceBoostMap: { 'skills/': 1.3 } }); + expect(a).not.toBe(b); + }); + + test('undefined sourceBoostMap is stable and distinct from an explicit map (legacy-caller fallback)', () => { + const k = baseKnobs(); + expect(knobsHash(k)).toBe(knobsHash(k)); + expect(knobsHash(k)).not.toBe( + knobsHash(k, { sourceBoostMap: resolveBoostMap(undefined) }), + ); + }); +});