Files
gbrain/test/search-alias-resolved-boost.test.ts
T
0bbaed2e48 v0.42.67.0 feat(migrate): provider-agnostic embedding migration — the path off ZeroEntropy (#3390, fixes #3391) (#3459)
* feat(migrate): provider-agnostic embedding migration service — the path off ZeroEntropy (#3390)

- gbrain migrate embeddings --to <provider:model> (alias: retrieval-upgrade):
  plan + cost preflight, consent gate (--yes / TTY confirm / non-TTY exit 2),
  live probe against the target provider before any mutation, env-override
  gate, schema dimension transition via the shared runSchemaTransition,
  dual-plane config write, NULL-signature-inclusive invalidation, query-cache
  purge, resumable re-embed through the standard embed pipeline (single-flight
  locks, backoff, pacing, stderr progress). Killed runs resume by re-running
  the same command; the NULL-embedding column is the checkpoint.
- #3391 root-cause fix (both engines): countStaleChunks / sumStaleChunkChars /
  invalidateStaleSignatureEmbeddings accept includeNullSignature to lift the
  v108 grandfather clause; embed --stale warns loudly when a model swap
  leaves NULL-signature pages in the old embedding space, and
  --include-null-signature re-embeds them. Default sweep behavior unchanged.
- knobs_hash v=12 → v=13 (prov=default legacy callers must not be served
  pre-migration cache rows).
- migrate_embeddings op: scope admin, localOnly, hidden cliHints, hard
  remote refusal, needs_confirmation without yes=true.
- One-shot post-upgrade ZE-sunset banner (ze_sunset_notice_shown) for brains
  resolving to a zeroentropyai:* embedding model or reranker.
- doctor's dimension-mismatch repair hint now names the real command.
- Docs: docs/guides/embedding-migration.md, KEY_FILES entries, spend-controls
  gate row. Tests: PGLite unit + full-lifecycle flow (interrupted-run resume),
  real-Postgres e2e (pgvector DDL path + #3391 predicate parity).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(test): satisfy check:test-isolation + bump the remaining knobs_hash pins

- test/migrate-embeddings-flow.test.ts → .serial.test.ts: the file holds a
  temp GBRAIN_HOME + an installed fake embed transport for its whole
  lifecycle (beforeAll → afterAll), which withEnv() can't wrap. This also
  fixes the CI shard-pollution failure in
  test/ai/recipes-existing-regression.test.ts (that file passes solo on both
  master and this branch; the flow test's configureGateway + provider-key
  deletion was leaking into it inside the same shard process).
- test/embedding-migration.test.ts: env-override case now uses withEnv().
- Bump the three remaining KNOBS_HASH_VERSION pins to 13
  (cross-modal-phase1, search-alias-resolved-boost, search/knobs-hash-reranker).
- Docs + llms bundles follow the test rename.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* chore(test): wire the new Postgres e2e into the smart e2e selector map

Changes to embed.ts / embedding-migration.ts / retrieval-upgrade-planner.ts /
postgres-engine.ts now trigger test/e2e/migrate-embeddings-postgres.test.ts —
the #3391 stale predicates and runSchemaTransition's DDL path behave
differently on real pgvector than on PGLite, so the smart selector has to know.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* feat(migrate): consult spend.posture in the embedding-migration consent gate

The brief asked the gate to honor spend.posture; it previously didn't read it
at all. Now it does — but deliberately does NOT bypass on tokenmax: posture
waives the spend CEILING, and this gate also guards a destructive schema
rebuild (existing vectors dropped, retrieval degraded until the re-embed
finishes). Under tokenmax the dollar figure is marked informational on stderr
and the confirmation is still asked; --yes stays the single scripted bypass.

Pinned by a new case in the flow test so a later refactor can't quietly turn
posture into a bypass. Guide + spend-controls table updated to match.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* wip: blocker fixes

---------

Co-authored-by: Garry Tan <garrytan@gmail.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
2026-07-27 17:48:37 -07:00

96 lines
3.8 KiB
TypeScript

// v0.42 Type Unification (T32) — alias_resolved search boost stage.
//
// Coverage: pages whose slug is a canonical_slug in slug_aliases get 1.05x
// score multiplier; non-alias-canonical pages unchanged; stage stamps
// alias_resolved_boost field for --explain; KNOBS_HASH_VERSION bumped.
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'bun:test';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { resetPgliteState } from './helpers/reset-pglite.ts';
import { runPostFusionStages, type PostFusionOpts } from '../src/core/search/hybrid.ts';
import { KNOBS_HASH_VERSION } from '../src/core/search/mode.ts';
import type { SearchResult } from '../src/core/types.ts';
let engine: PGLiteEngine;
beforeAll(async () => {
engine = new PGLiteEngine();
await engine.connect({});
await engine.initSchema();
});
afterAll(async () => {
await engine.disconnect();
});
beforeEach(async () => {
await resetPgliteState(engine);
});
const noopPostFusionOpts: PostFusionOpts = {
applyBacklinks: false,
salience: 'off',
recency: 'off',
graphSignalsEnabled: false,
};
describe('alias_resolved boost stage', () => {
it('applies 1.05x multiplier to pages that are canonicals of aliases', async () => {
// Insert an alias pointing at canonical-page
await engine.executeRaw(
`INSERT INTO slug_aliases (source_id, alias_slug, canonical_slug) VALUES ('default', 'old-name', 'canonical-page')`,
);
const results: SearchResult[] = [
{
slug: 'canonical-page', source_id: 'default', score: 1.0,
chunk_id: 1, page_id: 1, chunk_text: '', chunk_index: 0,
title: 'Canonical', type: 'concept' as never, slug_lower: 'canonical-page',
} as unknown as SearchResult,
{
slug: 'plain-page', source_id: 'default', score: 1.0,
chunk_id: 2, page_id: 2, chunk_text: '', chunk_index: 0,
title: 'Plain', type: 'concept' as never, slug_lower: 'plain-page',
} as unknown as SearchResult,
];
await runPostFusionStages(engine, results, noopPostFusionOpts);
// canonical-page gets 1.05x boost
expect(results[0].score).toBeCloseTo(1.05, 5);
expect(results[0].alias_resolved_boost).toBe(1.05);
// plain-page unchanged
expect(results[1].score).toBeCloseTo(1.0, 5);
expect(results[1].alias_resolved_boost).toBeUndefined();
});
it('does not boost when no aliases exist', async () => {
const results: SearchResult[] = [{
slug: 'plain', source_id: 'default', score: 1.0,
chunk_id: 1, page_id: 1, chunk_text: '', chunk_index: 0,
title: 'p', type: 'concept' as never, slug_lower: 'plain',
} as unknown as SearchResult];
await runPostFusionStages(engine, results, noopPostFusionOpts);
expect(results[0].score).toBeCloseTo(1.0, 5);
expect(results[0].alias_resolved_boost).toBeUndefined();
});
it('is source-scoped (F9): alias in source A does not boost in source B', async () => {
await engine.executeRaw(`INSERT INTO sources (id, name) VALUES ('alt', 'alt') ON CONFLICT DO NOTHING`);
await engine.executeRaw(
`INSERT INTO slug_aliases (source_id, alias_slug, canonical_slug) VALUES ('alt', 'old', 'shared')`,
);
// Same slug, different source — should NOT be boosted (alias is in 'alt')
const results: SearchResult[] = [{
slug: 'shared', source_id: 'default', score: 1.0,
chunk_id: 1, page_id: 1, chunk_text: '', chunk_index: 0,
title: 's', type: 'concept' as never, slug_lower: 'shared',
} as unknown as SearchResult];
await runPostFusionStages(engine, results, noopPostFusionOpts);
expect(results[0].alias_resolved_boost).toBeUndefined();
});
});
describe('KNOBS_HASH_VERSION', () => {
it('is 13 (12→13 embedding-provider migration invalidates rows written against the prior embedding space, #3390)', () => {
expect(KNOBS_HASH_VERSION).toBe(13);
});
});