From bb417051fe88ad4474620a72d4bd09d07063b1b1 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 16 Jul 2026 16:57:55 -0700 Subject: [PATCH] =?UTF-8?q?fix(search):=20fold=20hard-exclude/include=20pr?= =?UTF-8?q?efixes=20into=20knobs=5Fhash=20=E2=80=94=20stop=20cross-process?= =?UTF-8?q?=20cache=20leak=20of=20excluded=20slugs=20(#2825)=20(#2885)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resolveHardExcludes() only ran at DB-query build time (cache miss), so query_cache rows written by a process without GBRAIN_SEARCH_EXCLUDE could be served to a process with it (and vice versa), leaking excluded slugs. hybridSearchCached now resolves the effective hard-exclude list exactly as the engines' query-build path does and folds it (sorted, append-only hx= part) into knobsHash via a new KnobsHashContext.hardExcludes field. KNOBS_HASH_VERSION 11 -> 12: one-time global cache cold-miss on upgrade, refills within cache.ttl_seconds. Co-authored-by: Sinabina Co-authored-by: Claude Fable 5 --- src/core/search/hybrid.ts | 7 ++++ src/core/search/mode.ts | 27 ++++++++++++- test/cross-modal-phase1.test.ts | 6 +-- test/query-cache-knobs-hash.serial.test.ts | 47 ++++++++++++++++++++++ test/search-alias-resolved-boost.test.ts | 4 +- test/search-mode.test.ts | 9 +++-- test/search/knobs-hash-reranker.test.ts | 40 +++++++++++++++++- 7 files changed, 129 insertions(+), 11 deletions(-) diff --git a/src/core/search/hybrid.ts b/src/core/search/hybrid.ts index 7b451ea63..d4669a1cc 100644 --- a/src/core/search/hybrid.ts +++ b/src/core/search/hybrid.ts @@ -15,6 +15,7 @@ import type { SearchResult, SearchOpts, HybridSearchMeta } from '../types.ts'; import { embed, embedQuery } from '../embedding.ts'; import { registerBackgroundWorkDrainer } from '../background-work.ts'; import { resolveEmbeddingColumn, isCacheSafe } from './embedding-column.ts'; +import { resolveHardExcludes } from './source-boost.ts'; import { resolveAdaptiveReturn, applyAdaptiveReturn, @@ -1631,6 +1632,12 @@ export async function hybridSearchCached( const cacheKnobsHash = knobsHash(resolvedForCache, { embeddingColumn: resolvedColCached.name, embeddingModel: resolvedColCached.embeddingModel, + // #2825 — fold the resolved hard-exclude prefix list (defaults ∪ + // GBRAIN_SEARCH_EXCLUDE ∪ per-call exclude_slug_prefixes, minus + // include_slug_prefixes — exactly what the engines' query-build path + // resolves) into the cache key so a row written under one exclude + // policy can't be served to a lookup under another. + hardExcludes: resolveHardExcludes(opts?.exclude_slug_prefixes, opts?.include_slug_prefixes), }); // Cache decision: opts.useCache (explicit) wins over global config; global diff --git a/src/core/search/mode.ts b/src/core/search/mode.ts index f851913c0..cb5a14782 100644 --- a/src/core/search/mode.ts +++ b/src/core/search/mode.ts @@ -747,7 +747,16 @@ export function attributeKnob( // to post-fix lookups. Same one-time global cold-miss pattern as the bumps // above (the hash is global, not per-provider); refills within // cache.ttl_seconds (3600s default). -export const KNOBS_HASH_VERSION = 11; +// +// bump 11→12 (2026-07-16, #2825): the resolved hard-exclude slug-prefix list +// (defaults ∪ GBRAIN_SEARCH_EXCLUDE ∪ exclude_slug_prefixes, minus +// include_slug_prefixes) folds into the key via ctx.hardExcludes. It only +// applied at DB-query build time (cache miss), so a process with +// GBRAIN_SEARCH_EXCLUDE set could be served cached rows containing excluded +// slugs written by a process without it, and vice versa. Same one-time +// global cold-miss pattern as the bumps above; refills within +// cache.ttl_seconds (3600s default). +export const KNOBS_HASH_VERSION = 12; /** * v0.36 (D8 / CDX-2) — second-arg context for the cache key. The @@ -776,6 +785,16 @@ export interface KnobsHashContext { */ schemaPack?: string; schemaPackVersion?: string; + /** + * v=12 (#2825): the RESOLVED effective hard-exclude prefix list — the same + * value resolveHardExcludes() produces at query-build time (defaults ∪ + * GBRAIN_SEARCH_EXCLUDE ∪ per-call exclude_slug_prefixes, minus + * include_slug_prefixes). Folded (sorted, so input order is irrelevant) + * into the hash so a cache row written under one exclude policy can never + * be served to a lookup under another. Undefined falls back to the literal + * 'none' for legacy callers that don't thread excludes. + */ + hardExcludes?: string[]; } export function knobsHash( @@ -863,6 +882,12 @@ export function knobsHash( // test/model-pricing.test.ts-style drift guards and the mode tests. `rel=${knobs.relationalRetrieval ? 1 : 0}`, `reld=${knobs.relational_retrieval_depth ?? 2}`, + // v=12 addition (#2825, append-only): resolved hard-exclude prefixes. + // Before this, resolveHardExcludes() only ran at DB-query build time + // (cache miss), so cached rows leaked GBRAIN_SEARCH_EXCLUDE'd slugs + // across processes. Sorted copy so ['a/','b/'] and ['b/','a/'] hash + // identically; undefined falls back to 'none' for legacy callers. + `hx=${ctx?.hardExcludes ? [...ctx.hardExcludes].sort().join(',') : 'none'}`, ]; const h = createHash('sha256'); h.update(parts.join('|')); diff --git a/test/cross-modal-phase1.test.ts b/test/cross-modal-phase1.test.ts index 4f5243d96..19689a324 100644 --- a/test/cross-modal-phase1.test.ts +++ b/test/cross-modal-phase1.test.ts @@ -136,7 +136,7 @@ describe('D2 — knobsHash differs across cross-modal knob values', () => { return resolveSearchMode({ mode: 'balanced' }); } - test('KNOBS_HASH_VERSION is 11 (cross-modal still appended; 10→11 asymmetric input_type fix)', () => { + test('KNOBS_HASH_VERSION is 12 (cross-modal still appended; 11→12 hard-exclude fold #2825)', () => { // v0.35 ladder: 1→2 reranker, 2→3 floor_ratio. v0.36 piggybacks on v=3 // with 7 cross-modal knobs + column/provider context. v0.40.4 (salem) + // v0.39 T21 (master) bump to v=4 for graph_signals + schema-pack fields. @@ -145,8 +145,8 @@ describe('D2 — knobsHash differs across cross-modal knob values', () => { // T2: 6→7 title_boost. v0.42.3.0: 7→8 autocut. issue #1777: 8→9 archive/ demote. // v0.43: 9→10 relational recall arm. #1400: 10→11 query-side input_type // finally reaches asymmetric providers — pre-fix rows were keyed on - // document-side query vectors. - expect(KNOBS_HASH_VERSION).toBe(11); + // document-side query vectors. #2825: 11→12 hard-exclude fold (hx=). + expect(KNOBS_HASH_VERSION).toBe(12); }); test('flipping unified_multimodal changes the hash', () => { diff --git a/test/query-cache-knobs-hash.serial.test.ts b/test/query-cache-knobs-hash.serial.test.ts index fdfc02745..d11d42ef3 100644 --- a/test/query-cache-knobs-hash.serial.test.ts +++ b/test/query-cache-knobs-hash.serial.test.ts @@ -19,6 +19,7 @@ import { PGLiteEngine } from '../src/core/pglite-engine.ts'; import { SemanticQueryCache, cacheRowId } from '../src/core/search/query-cache.ts'; import type { SearchResult } from '../src/core/types.ts'; import { knobsHash, resolveSearchMode } from '../src/core/search/mode.ts'; +import { resolveHardExcludes } from '../src/core/search/source-boost.ts'; import { configureGateway, resetGateway } from '../src/core/ai/gateway.ts'; let engine: PGLiteEngine; @@ -230,3 +231,49 @@ describe('SemanticQueryCache cross-mode isolation (CDX-4 hotfix)', () => { expect(conservativeHit.hit).toBe(false); }); }); + +describe('hard-exclude cache isolation (#2825)', () => { + // Hashes computed the way hybridSearchCached does: same resolved mode, ctx + // carrying the resolved hard-exclude list. A row written by a process + // WITHOUT GBRAIN_SEARCH_EXCLUDE (defaults only) must not be served to a + // process WITH it, and vice versa. + const noEnvHash = knobsHash(resolveSearchMode({ mode: 'balanced' }), { + hardExcludes: resolveHardExcludes(undefined, undefined, undefined), + }); + const envExcludeHash = knobsHash(resolveSearchMode({ mode: 'balanced' }), { + hardExcludes: resolveHardExcludes(undefined, undefined, 'private/'), + }); + + test('row written without excludes is NOT served to a lookup with excludes', async () => { + const cache = new SemanticQueryCache(engine); + const emb = makeEmbedding(6); + + // Simulate a no-exclude process writing results that include a slug the + // excluding process must never see. + const leaky = makeResults('private', 5); + await cache.store('who is alice', emb, leaky, { + vector_enabled: true, detail_resolved: null, expansion_applied: false, + }, { knobsHash: noEnvHash }); + + // Excluding process → MISS (falls through to a fresh, filtered query). + const excluded = await cache.lookup(emb, { knobsHash: envExcludeHash }); + expect(excluded.hit).toBe(false); + + // Original no-exclude process still hits its own row. + const original = await cache.lookup(emb, { knobsHash: noEnvHash }); + expect(original.hit).toBe(true); + expect(original.results?.length).toBe(5); + }); + + test('row written WITH excludes is not served back once excludes are lifted', async () => { + const cache = new SemanticQueryCache(engine); + const emb = makeEmbedding(7); + + await cache.store('who is alice', emb, makeResults('filtered', 3), { + vector_enabled: true, detail_resolved: null, expansion_applied: false, + }, { knobsHash: envExcludeHash }); + + expect((await cache.lookup(emb, { knobsHash: noEnvHash })).hit).toBe(false); + expect((await cache.lookup(emb, { knobsHash: envExcludeHash })).hit).toBe(true); + }); +}); diff --git a/test/search-alias-resolved-boost.test.ts b/test/search-alias-resolved-boost.test.ts index 57fe7ace0..224d2d988 100644 --- a/test/search-alias-resolved-boost.test.ts +++ b/test/search-alias-resolved-boost.test.ts @@ -89,7 +89,7 @@ describe('alias_resolved boost stage', () => { }); describe('KNOBS_HASH_VERSION', () => { - it('is 11 (10→11 asymmetric input_type fix invalidates document-side query-vector rows, #1400)', () => { - expect(KNOBS_HASH_VERSION).toBe(11); + it('is 12 (11→12 hard-exclude fold invalidates rows written under a different exclude policy, #2825)', () => { + expect(KNOBS_HASH_VERSION).toBe(12); }); }); diff --git a/test/search-mode.test.ts b/test/search-mode.test.ts index a8e9ecf18..223d112d9 100644 --- a/test/search-mode.test.ts +++ b/test/search-mode.test.ts @@ -407,7 +407,10 @@ describe('knobsHash determinism + cross-mode separation (CDX-4)', () => { // now produces query-side vectors for asymmetric providers (zembed-1, // Voyage v3+), so rows keyed on pre-fix document-side query vectors // must not be served to post-fix lookups. - expect(KNOBS_HASH_VERSION).toBe(11); + // #2825: bumped 11→12 to fold the resolved hard-exclude prefix list + // (hx=) — cached rows leaked GBRAIN_SEARCH_EXCLUDE'd slugs across + // processes. + expect(KNOBS_HASH_VERSION).toBe(12); }); test('T1 (codex): floor_ratio set vs unset produces DIFFERENT hashes (cache contamination prevention)', () => { @@ -572,8 +575,8 @@ describe('v0.40.4 — graph_signals knob', () => { }); describe('v0.42.3.0 — autocut knobs', () => { - test('KNOBS_HASH_VERSION is 11 (10→11 asymmetric input_type fix, #1400)', () => { - expect(KNOBS_HASH_VERSION).toBe(11); + test('KNOBS_HASH_VERSION is 12 (11→12 hard-exclude fold, #2825)', () => { + expect(KNOBS_HASH_VERSION).toBe(12); }); test('bundle defaults: conservative off, balanced/tokenmax on @0.20', () => { diff --git a/test/search/knobs-hash-reranker.test.ts b/test/search/knobs-hash-reranker.test.ts index bd793d427..9f73ac394 100644 --- a/test/search/knobs-hash-reranker.test.ts +++ b/test/search/knobs-hash-reranker.test.ts @@ -27,6 +27,7 @@ import { MODE_BUNDLES, type ResolvedSearchKnobs, } from '../../src/core/search/mode.ts'; +import { resolveHardExcludes } from '../../src/core/search/source-boost.ts'; /** Build a baseline resolved knob set with all reranker fields filled. */ function baseKnobs(): ResolvedSearchKnobs { @@ -43,7 +44,7 @@ function baseKnobs(): ResolvedSearchKnobs { } describe('KNOBS_HASH_VERSION + version invariants', () => { - test('version is 11 (…; 8→9 archive-demote #1777; 9→10 relational recall; 10→11 asymmetric input_type #1400)', () => { + test('version is 12 (…; 9→10 relational recall; 10→11 asymmetric input_type #1400; 11→12 hard-excludes #2825)', () => { // v0.35.0.0: 1→2 to fold reranker fields. v0.35.6.0: 2→3 to fold // floor_ratio. v0.36 wave: piggybacks on v=3 with 7 cross-modal knobs // (D2) PLUS column + provider context (D8/CDX-2 cross-column isolation). @@ -61,7 +62,9 @@ describe('KNOBS_HASH_VERSION + version invariants', () => { // #1400: 10→11 asymmetric input_type fix — embedQuery() now produces // query-side vectors for asymmetric providers, so rows keyed on // pre-fix document-side query vectors must not be served. - expect(KNOBS_HASH_VERSION).toBe(11); + // #2825: 11→12 to fold the resolved hard-exclude prefix list (hx=) — + // cached rows leaked GBRAIN_SEARCH_EXCLUDE'd slugs across processes. + expect(KNOBS_HASH_VERSION).toBe(12); }); test('hash is 16 hex chars regardless of reranker config', () => { @@ -208,3 +211,36 @@ describe('append-only convention (CDX2-F13)', () => { expect(bare).toBe(explicit); }); }); + +describe('v=12 hard-exclude participation (#2825)', () => { + test('different exclude lists → different hashes', () => { + const k = baseKnobs(); + const noEnv = knobsHash(k, { hardExcludes: resolveHardExcludes(undefined, undefined, undefined) }); + const withEnv = knobsHash(k, { hardExcludes: resolveHardExcludes(undefined, undefined, 'private/') }); + expect(noEnv).not.toBe(withEnv); + }); + + test('include (opt-back-in) changes the hash too', () => { + const k = baseKnobs(); + const a = knobsHash(k, { hardExcludes: resolveHardExcludes(undefined, undefined, undefined) }); + const b = knobsHash(k, { hardExcludes: resolveHardExcludes(undefined, ['test/'], undefined) }); + expect(a).not.toBe(b); + }); + + test('same prefixes in different input order → SAME hash (normalization)', () => { + const k = baseKnobs(); + const a = knobsHash(k, { hardExcludes: ['a/', 'b/', 'test/'] }); + const b = knobsHash(k, { hardExcludes: ['test/', 'b/', 'a/'] }); + expect(a).toBe(b); + }); + + test('undefined hardExcludes is stable (legacy-caller fallback)', () => { + const k = baseKnobs(); + expect(knobsHash(k)).toBe(knobsHash(k)); + // ...and distinct from an explicit resolved default list — a legacy + // caller can never collide with a policy-carrying cache row. + expect(knobsHash(k)).not.toBe( + knobsHash(k, { hardExcludes: resolveHardExcludes(undefined, undefined, undefined) }), + ); + }); +});