mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
fix(search): fold hard-exclude/include prefixes into knobs_hash — stop cross-process cache leak of excluded slugs (#2825) (#2885)
resolveHardExcludes() only ran at DB-query build time (cache miss), so query_cache rows written by a process without GBRAIN_SEARCH_EXCLUDE could be served to a process with it (and vice versa), leaking excluded slugs. hybridSearchCached now resolves the effective hard-exclude list exactly as the engines' query-build path does and folds it (sorted, append-only hx= part) into knobsHash via a new KnobsHashContext.hardExcludes field. KNOBS_HASH_VERSION 11 -> 12: one-time global cache cold-miss on upgrade, refills within cache.ttl_seconds. Co-authored-by: Sinabina <sinabina@Sinabinas-MacBook-Pro-4.local> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Sinabina
Claude Fable 5
parent
86962b242b
commit
bb417051fe
@@ -15,6 +15,7 @@ import type { SearchResult, SearchOpts, HybridSearchMeta } from '../types.ts';
|
||||
import { embed, embedQuery } from '../embedding.ts';
|
||||
import { registerBackgroundWorkDrainer } from '../background-work.ts';
|
||||
import { resolveEmbeddingColumn, isCacheSafe } from './embedding-column.ts';
|
||||
import { resolveHardExcludes } from './source-boost.ts';
|
||||
import {
|
||||
resolveAdaptiveReturn,
|
||||
applyAdaptiveReturn,
|
||||
@@ -1631,6 +1632,12 @@ export async function hybridSearchCached(
|
||||
const cacheKnobsHash = knobsHash(resolvedForCache, {
|
||||
embeddingColumn: resolvedColCached.name,
|
||||
embeddingModel: resolvedColCached.embeddingModel,
|
||||
// #2825 — fold the resolved hard-exclude prefix list (defaults ∪
|
||||
// GBRAIN_SEARCH_EXCLUDE ∪ per-call exclude_slug_prefixes, minus
|
||||
// include_slug_prefixes — exactly what the engines' query-build path
|
||||
// resolves) into the cache key so a row written under one exclude
|
||||
// policy can't be served to a lookup under another.
|
||||
hardExcludes: resolveHardExcludes(opts?.exclude_slug_prefixes, opts?.include_slug_prefixes),
|
||||
});
|
||||
|
||||
// Cache decision: opts.useCache (explicit) wins over global config; global
|
||||
|
||||
+26
-1
@@ -747,7 +747,16 @@ export function attributeKnob<K extends keyof ModeBundle>(
|
||||
// to post-fix lookups. Same one-time global cold-miss pattern as the bumps
|
||||
// above (the hash is global, not per-provider); refills within
|
||||
// cache.ttl_seconds (3600s default).
|
||||
export const KNOBS_HASH_VERSION = 11;
|
||||
//
|
||||
// bump 11→12 (2026-07-16, #2825): the resolved hard-exclude slug-prefix list
|
||||
// (defaults ∪ GBRAIN_SEARCH_EXCLUDE ∪ exclude_slug_prefixes, minus
|
||||
// include_slug_prefixes) folds into the key via ctx.hardExcludes. It only
|
||||
// applied at DB-query build time (cache miss), so a process with
|
||||
// GBRAIN_SEARCH_EXCLUDE set could be served cached rows containing excluded
|
||||
// slugs written by a process without it, and vice versa. Same one-time
|
||||
// global cold-miss pattern as the bumps above; refills within
|
||||
// cache.ttl_seconds (3600s default).
|
||||
export const KNOBS_HASH_VERSION = 12;
|
||||
|
||||
/**
|
||||
* v0.36 (D8 / CDX-2) — second-arg context for the cache key. The
|
||||
@@ -776,6 +785,16 @@ export interface KnobsHashContext {
|
||||
*/
|
||||
schemaPack?: string;
|
||||
schemaPackVersion?: string;
|
||||
/**
|
||||
* v=12 (#2825): the RESOLVED effective hard-exclude prefix list — the same
|
||||
* value resolveHardExcludes() produces at query-build time (defaults ∪
|
||||
* GBRAIN_SEARCH_EXCLUDE ∪ per-call exclude_slug_prefixes, minus
|
||||
* include_slug_prefixes). Folded (sorted, so input order is irrelevant)
|
||||
* into the hash so a cache row written under one exclude policy can never
|
||||
* be served to a lookup under another. Undefined falls back to the literal
|
||||
* 'none' for legacy callers that don't thread excludes.
|
||||
*/
|
||||
hardExcludes?: string[];
|
||||
}
|
||||
|
||||
export function knobsHash(
|
||||
@@ -863,6 +882,12 @@ export function knobsHash(
|
||||
// test/model-pricing.test.ts-style drift guards and the mode tests.
|
||||
`rel=${knobs.relationalRetrieval ? 1 : 0}`,
|
||||
`reld=${knobs.relational_retrieval_depth ?? 2}`,
|
||||
// v=12 addition (#2825, append-only): resolved hard-exclude prefixes.
|
||||
// Before this, resolveHardExcludes() only ran at DB-query build time
|
||||
// (cache miss), so cached rows leaked GBRAIN_SEARCH_EXCLUDE'd slugs
|
||||
// across processes. Sorted copy so ['a/','b/'] and ['b/','a/'] hash
|
||||
// identically; undefined falls back to 'none' for legacy callers.
|
||||
`hx=${ctx?.hardExcludes ? [...ctx.hardExcludes].sort().join(',') : 'none'}`,
|
||||
];
|
||||
const h = createHash('sha256');
|
||||
h.update(parts.join('|'));
|
||||
|
||||
@@ -136,7 +136,7 @@ describe('D2 — knobsHash differs across cross-modal knob values', () => {
|
||||
return resolveSearchMode({ mode: 'balanced' });
|
||||
}
|
||||
|
||||
test('KNOBS_HASH_VERSION is 11 (cross-modal still appended; 10→11 asymmetric input_type fix)', () => {
|
||||
test('KNOBS_HASH_VERSION is 12 (cross-modal still appended; 11→12 hard-exclude fold #2825)', () => {
|
||||
// v0.35 ladder: 1→2 reranker, 2→3 floor_ratio. v0.36 piggybacks on v=3
|
||||
// with 7 cross-modal knobs + column/provider context. v0.40.4 (salem) +
|
||||
// v0.39 T21 (master) bump to v=4 for graph_signals + schema-pack fields.
|
||||
@@ -145,8 +145,8 @@ describe('D2 — knobsHash differs across cross-modal knob values', () => {
|
||||
// T2: 6→7 title_boost. v0.42.3.0: 7→8 autocut. issue #1777: 8→9 archive/ demote.
|
||||
// v0.43: 9→10 relational recall arm. #1400: 10→11 query-side input_type
|
||||
// finally reaches asymmetric providers — pre-fix rows were keyed on
|
||||
// document-side query vectors.
|
||||
expect(KNOBS_HASH_VERSION).toBe(11);
|
||||
// document-side query vectors. #2825: 11→12 hard-exclude fold (hx=).
|
||||
expect(KNOBS_HASH_VERSION).toBe(12);
|
||||
});
|
||||
|
||||
test('flipping unified_multimodal changes the hash', () => {
|
||||
|
||||
@@ -19,6 +19,7 @@ import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { SemanticQueryCache, cacheRowId } from '../src/core/search/query-cache.ts';
|
||||
import type { SearchResult } from '../src/core/types.ts';
|
||||
import { knobsHash, resolveSearchMode } from '../src/core/search/mode.ts';
|
||||
import { resolveHardExcludes } from '../src/core/search/source-boost.ts';
|
||||
import { configureGateway, resetGateway } from '../src/core/ai/gateway.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
@@ -230,3 +231,49 @@ describe('SemanticQueryCache cross-mode isolation (CDX-4 hotfix)', () => {
|
||||
expect(conservativeHit.hit).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('hard-exclude cache isolation (#2825)', () => {
|
||||
// Hashes computed the way hybridSearchCached does: same resolved mode, ctx
|
||||
// carrying the resolved hard-exclude list. A row written by a process
|
||||
// WITHOUT GBRAIN_SEARCH_EXCLUDE (defaults only) must not be served to a
|
||||
// process WITH it, and vice versa.
|
||||
const noEnvHash = knobsHash(resolveSearchMode({ mode: 'balanced' }), {
|
||||
hardExcludes: resolveHardExcludes(undefined, undefined, undefined),
|
||||
});
|
||||
const envExcludeHash = knobsHash(resolveSearchMode({ mode: 'balanced' }), {
|
||||
hardExcludes: resolveHardExcludes(undefined, undefined, 'private/'),
|
||||
});
|
||||
|
||||
test('row written without excludes is NOT served to a lookup with excludes', async () => {
|
||||
const cache = new SemanticQueryCache(engine);
|
||||
const emb = makeEmbedding(6);
|
||||
|
||||
// Simulate a no-exclude process writing results that include a slug the
|
||||
// excluding process must never see.
|
||||
const leaky = makeResults('private', 5);
|
||||
await cache.store('who is alice', emb, leaky, {
|
||||
vector_enabled: true, detail_resolved: null, expansion_applied: false,
|
||||
}, { knobsHash: noEnvHash });
|
||||
|
||||
// Excluding process → MISS (falls through to a fresh, filtered query).
|
||||
const excluded = await cache.lookup(emb, { knobsHash: envExcludeHash });
|
||||
expect(excluded.hit).toBe(false);
|
||||
|
||||
// Original no-exclude process still hits its own row.
|
||||
const original = await cache.lookup(emb, { knobsHash: noEnvHash });
|
||||
expect(original.hit).toBe(true);
|
||||
expect(original.results?.length).toBe(5);
|
||||
});
|
||||
|
||||
test('row written WITH excludes is not served back once excludes are lifted', async () => {
|
||||
const cache = new SemanticQueryCache(engine);
|
||||
const emb = makeEmbedding(7);
|
||||
|
||||
await cache.store('who is alice', emb, makeResults('filtered', 3), {
|
||||
vector_enabled: true, detail_resolved: null, expansion_applied: false,
|
||||
}, { knobsHash: envExcludeHash });
|
||||
|
||||
expect((await cache.lookup(emb, { knobsHash: noEnvHash })).hit).toBe(false);
|
||||
expect((await cache.lookup(emb, { knobsHash: envExcludeHash })).hit).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -89,7 +89,7 @@ describe('alias_resolved boost stage', () => {
|
||||
});
|
||||
|
||||
describe('KNOBS_HASH_VERSION', () => {
|
||||
it('is 11 (10→11 asymmetric input_type fix invalidates document-side query-vector rows, #1400)', () => {
|
||||
expect(KNOBS_HASH_VERSION).toBe(11);
|
||||
it('is 12 (11→12 hard-exclude fold invalidates rows written under a different exclude policy, #2825)', () => {
|
||||
expect(KNOBS_HASH_VERSION).toBe(12);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -407,7 +407,10 @@ describe('knobsHash determinism + cross-mode separation (CDX-4)', () => {
|
||||
// now produces query-side vectors for asymmetric providers (zembed-1,
|
||||
// Voyage v3+), so rows keyed on pre-fix document-side query vectors
|
||||
// must not be served to post-fix lookups.
|
||||
expect(KNOBS_HASH_VERSION).toBe(11);
|
||||
// #2825: bumped 11→12 to fold the resolved hard-exclude prefix list
|
||||
// (hx=) — cached rows leaked GBRAIN_SEARCH_EXCLUDE'd slugs across
|
||||
// processes.
|
||||
expect(KNOBS_HASH_VERSION).toBe(12);
|
||||
});
|
||||
|
||||
test('T1 (codex): floor_ratio set vs unset produces DIFFERENT hashes (cache contamination prevention)', () => {
|
||||
@@ -572,8 +575,8 @@ describe('v0.40.4 — graph_signals knob', () => {
|
||||
});
|
||||
|
||||
describe('v0.42.3.0 — autocut knobs', () => {
|
||||
test('KNOBS_HASH_VERSION is 11 (10→11 asymmetric input_type fix, #1400)', () => {
|
||||
expect(KNOBS_HASH_VERSION).toBe(11);
|
||||
test('KNOBS_HASH_VERSION is 12 (11→12 hard-exclude fold, #2825)', () => {
|
||||
expect(KNOBS_HASH_VERSION).toBe(12);
|
||||
});
|
||||
|
||||
test('bundle defaults: conservative off, balanced/tokenmax on @0.20', () => {
|
||||
|
||||
@@ -27,6 +27,7 @@ import {
|
||||
MODE_BUNDLES,
|
||||
type ResolvedSearchKnobs,
|
||||
} from '../../src/core/search/mode.ts';
|
||||
import { resolveHardExcludes } from '../../src/core/search/source-boost.ts';
|
||||
|
||||
/** Build a baseline resolved knob set with all reranker fields filled. */
|
||||
function baseKnobs(): ResolvedSearchKnobs {
|
||||
@@ -43,7 +44,7 @@ function baseKnobs(): ResolvedSearchKnobs {
|
||||
}
|
||||
|
||||
describe('KNOBS_HASH_VERSION + version invariants', () => {
|
||||
test('version is 11 (…; 8→9 archive-demote #1777; 9→10 relational recall; 10→11 asymmetric input_type #1400)', () => {
|
||||
test('version is 12 (…; 9→10 relational recall; 10→11 asymmetric input_type #1400; 11→12 hard-excludes #2825)', () => {
|
||||
// v0.35.0.0: 1→2 to fold reranker fields. v0.35.6.0: 2→3 to fold
|
||||
// floor_ratio. v0.36 wave: piggybacks on v=3 with 7 cross-modal knobs
|
||||
// (D2) PLUS column + provider context (D8/CDX-2 cross-column isolation).
|
||||
@@ -61,7 +62,9 @@ describe('KNOBS_HASH_VERSION + version invariants', () => {
|
||||
// #1400: 10→11 asymmetric input_type fix — embedQuery() now produces
|
||||
// query-side vectors for asymmetric providers, so rows keyed on
|
||||
// pre-fix document-side query vectors must not be served.
|
||||
expect(KNOBS_HASH_VERSION).toBe(11);
|
||||
// #2825: 11→12 to fold the resolved hard-exclude prefix list (hx=) —
|
||||
// cached rows leaked GBRAIN_SEARCH_EXCLUDE'd slugs across processes.
|
||||
expect(KNOBS_HASH_VERSION).toBe(12);
|
||||
});
|
||||
|
||||
test('hash is 16 hex chars regardless of reranker config', () => {
|
||||
@@ -208,3 +211,36 @@ describe('append-only convention (CDX2-F13)', () => {
|
||||
expect(bare).toBe(explicit);
|
||||
});
|
||||
});
|
||||
|
||||
describe('v=12 hard-exclude participation (#2825)', () => {
|
||||
test('different exclude lists → different hashes', () => {
|
||||
const k = baseKnobs();
|
||||
const noEnv = knobsHash(k, { hardExcludes: resolveHardExcludes(undefined, undefined, undefined) });
|
||||
const withEnv = knobsHash(k, { hardExcludes: resolveHardExcludes(undefined, undefined, 'private/') });
|
||||
expect(noEnv).not.toBe(withEnv);
|
||||
});
|
||||
|
||||
test('include (opt-back-in) changes the hash too', () => {
|
||||
const k = baseKnobs();
|
||||
const a = knobsHash(k, { hardExcludes: resolveHardExcludes(undefined, undefined, undefined) });
|
||||
const b = knobsHash(k, { hardExcludes: resolveHardExcludes(undefined, ['test/'], undefined) });
|
||||
expect(a).not.toBe(b);
|
||||
});
|
||||
|
||||
test('same prefixes in different input order → SAME hash (normalization)', () => {
|
||||
const k = baseKnobs();
|
||||
const a = knobsHash(k, { hardExcludes: ['a/', 'b/', 'test/'] });
|
||||
const b = knobsHash(k, { hardExcludes: ['test/', 'b/', 'a/'] });
|
||||
expect(a).toBe(b);
|
||||
});
|
||||
|
||||
test('undefined hardExcludes is stable (legacy-caller fallback)', () => {
|
||||
const k = baseKnobs();
|
||||
expect(knobsHash(k)).toBe(knobsHash(k));
|
||||
// ...and distinct from an explicit resolved default list — a legacy
|
||||
// caller can never collide with a policy-carrying cache row.
|
||||
expect(knobsHash(k)).not.toBe(
|
||||
knobsHash(k, { hardExcludes: resolveHardExcludes(undefined, undefined, undefined) }),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user