mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-30 03:12:32 +00:00
fix(embed): stamp the real model on chunk provenance and keep contextual prefixes on --stale (#3461, #3507) (#3538)
#3461 — insertChunks stamped DEFAULT_EMBEDDING_MODEL on chunk provenance
whenever the AI gateway was unconfigured: getEmbeddingModel() throws
rather than returning falsy, so the reland's '|| resolvedModel' guard
(e1919fab) was dead code and the catch path kept the compile-time
constant. Both engines now fall back to the brain's own
config.embedding_model row on the throw path, with the compiled default
as last resort. Same-line sibling: the ON CONFLICT clause overwrote
'model' via COALESCE even when the existing vector was preserved —
'model' now mirrors the 'embedding' CASE branch-for-branch so the label
always describes whichever vector wins the upsert. The initSchema
sizing sites drop their dead '||' terms too.
#3507 — every plain re-embed path (embed <slug>, --all, --stale, and
the embed-backfill Minion loop) embedded raw chunk_text, silently
stripping the contextual-retrieval prefixes the sync path applied —
and embed --stale is the NORMAL post-model-migration path. All four
sites now wrap through wrapChunkTextsForStoredMode(), reproducing the
page's STORED convention (pages.contextual_retrieval_mode):
title/per_chunk_synopsis pages get the title-tier prefix, fenced_code
chunks stay unwrapped (D20-T4), unstamped pages stay raw. A fully
re-embedded per_chunk_synopsis page is restamped to 'title' so the
mode column keeps describing the vectors actually in the DB.
Deliberately NOT folding the contextual mode into
currentEmbeddingSignature(): the convention is already recorded
per-page (mode + corpus_generation), pages legitimately differ
per-page so a global signature cannot represent it, and bumping
signature semantics would force a full-corpus re-embed on upgrade.
No KNOBS_HASH_VERSION change (no collision with #3514).
Tests fail on unmodified master (6) and pass with the fix; postgres.js
path verified against a real pgvector instance in addition to the
PGLite suite.
Co-authored-by: Garry Tan <garrytan@gmail.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Garry Tan
Claude Opus 5
parent
539d015cc5
commit
176836f84d
+50
-3
@@ -19,6 +19,26 @@ import {
|
||||
} from '../core/pace-mode.ts';
|
||||
import { tryAcquireDbLock, type DbLockHandle } from '../core/db-lock.ts';
|
||||
import { embedBackfillLockId } from '../core/embed-backfill-lock.ts';
|
||||
import { wrapChunkTextsForStoredMode } from '../core/embedding-context.ts';
|
||||
import { titleTierCorpusGeneration } from '../core/contextual-retrieval-service.ts';
|
||||
import type { Page } from '../core/types.ts';
|
||||
|
||||
/**
|
||||
* #3507 — after a plain re-embed fully re-embedded a `per_chunk_synopsis`
|
||||
* page at the title-only tier (see wrapChunkTextsForStoredMode), restamp the
|
||||
* page's CR state to 'title' so `contextual_retrieval_mode` keeps describing
|
||||
* the vectors actually in the column. The reindex sweep restores the synopsis
|
||||
* tier later. No-op for every other mode.
|
||||
*/
|
||||
export async function restampIfDemotedToTitleTier(
|
||||
engine: BrainEngine,
|
||||
page: Pick<Page, 'contextual_retrieval_mode'> | null | undefined,
|
||||
slug: string,
|
||||
sourceId: string,
|
||||
): Promise<void> {
|
||||
if (page?.contextual_retrieval_mode !== 'per_chunk_synopsis') return;
|
||||
await engine.updatePageContextualRetrievalState(slug, sourceId, 'title', titleTierCorpusGeneration());
|
||||
}
|
||||
|
||||
export interface EmbedOpts {
|
||||
/** Embed ALL pages (every chunk). */
|
||||
@@ -599,7 +619,11 @@ async function embedPage(
|
||||
return;
|
||||
}
|
||||
|
||||
const embeddings = await embedBatch(toEmbed.map(c => c.chunk_text), { abortSignal: signal });
|
||||
// #3507: embed with the page's STORED wrapping convention (title-tier
|
||||
// contextual prefix when the page was embedded wrapped), not raw
|
||||
// chunk_text — otherwise a re-embed silently strips the contextual
|
||||
// prefixes the sync path applied. fenced_code chunks stay unwrapped.
|
||||
const embeddings = await embedBatch(wrapChunkTextsForStoredMode(page, toEmbed), { abortSignal: signal });
|
||||
const embeddingMap = new Map<number, Float32Array>();
|
||||
for (let j = 0; j < toEmbed.length; j++) {
|
||||
embeddingMap.set(toEmbed[j].chunk_index, embeddings[j]);
|
||||
@@ -622,6 +646,9 @@ async function embedPage(
|
||||
// such a page and then stamps it.
|
||||
if (toEmbed.length === chunks.length) {
|
||||
await engine.setPageEmbeddingSignature(slug, { sourceId, signature: currentEmbeddingSignature() });
|
||||
// #3507: a fully re-embedded per_chunk_synopsis page landed at the
|
||||
// title tier — keep the stamped mode honest.
|
||||
await restampIfDemotedToTitleTier(engine, page, slug, page.source_id);
|
||||
}
|
||||
result.embedded += toEmbed.length;
|
||||
result.pages_processed++;
|
||||
@@ -763,7 +790,8 @@ async function embedAll(
|
||||
}
|
||||
|
||||
try {
|
||||
const embeddings = await embedBatch(toEmbed.map(c => c.chunk_text));
|
||||
// #3507: reproduce the page's stored wrapping convention (see embedPage).
|
||||
const embeddings = await embedBatch(wrapChunkTextsForStoredMode(page, toEmbed));
|
||||
// Build a map of new embeddings by chunk_index
|
||||
const embeddingMap = new Map<number, Float32Array>();
|
||||
for (let j = 0; j < toEmbed.length; j++) {
|
||||
@@ -785,6 +813,11 @@ async function embedAll(
|
||||
await observed(pacer, () =>
|
||||
engine.setPageEmbeddingSignature(page.slug, { sourceId: pageSourceId, signature }),
|
||||
);
|
||||
// #3507: --all fully re-embeds; a per_chunk_synopsis page landed at
|
||||
// the title tier — keep the stamped mode honest.
|
||||
await observed(pacer, () =>
|
||||
restampIfDemotedToTitleTier(engine, page, page.slug, pageSourceId),
|
||||
);
|
||||
result.embedded += toEmbed.length;
|
||||
} catch (e: unknown) {
|
||||
serr(`\n Error embedding ${page.slug}: ${e instanceof Error ? e.message : e}`);
|
||||
@@ -1098,7 +1131,13 @@ async function embedAllStale(
|
||||
const keySourceId = stale[0]?.source_id ?? 'default';
|
||||
const slug = stale[0].slug;
|
||||
try {
|
||||
const embeddings = await embedBatchWithBackoff(stale.map(c => c.chunk_text), { abortSignal: effectiveSignal });
|
||||
// #3507: fetch the page row for its title + stored CR mode so the
|
||||
// re-embed reproduces the page's wrapping convention instead of
|
||||
// silently stripping contextual prefixes — `embed --stale` is the
|
||||
// NORMAL post-model-migration path, so raw-text embedding here
|
||||
// quietly converted whole corpora to the unwrapped convention.
|
||||
const pageRow = await observed(pacer, () => engine.getPage(slug, { sourceId: keySourceId }));
|
||||
const embeddings = await embedBatchWithBackoff(wrapChunkTextsForStoredMode(pageRow, stale), { abortSignal: effectiveSignal });
|
||||
// Re-fetch existing chunks and merge to avoid deleting non-stale chunks.
|
||||
const existing = await observed(pacer, () => engine.getChunks(slug, { sourceId: keySourceId }));
|
||||
const staleIdxToEmbedding = new Map<number, Float32Array>();
|
||||
@@ -1126,6 +1165,14 @@ async function embedAllStale(
|
||||
engine.setPageEmbeddingSignature(slug, { sourceId: keySourceId, signature }),
|
||||
);
|
||||
}
|
||||
// #3507: a FULLY re-embedded per_chunk_synopsis page landed at the
|
||||
// title tier — keep the stamped mode honest. Partially-stale pages
|
||||
// stay stamped as-is (mixed provenance; reindex sweeps fix them).
|
||||
if (stale.length === existing.length) {
|
||||
await observed(pacer, () =>
|
||||
restampIfDemotedToTitleTier(engine, pageRow, slug, keySourceId),
|
||||
);
|
||||
}
|
||||
result.embedded += stale.length;
|
||||
} catch (e: unknown) {
|
||||
// Budget/abort-fired cancellations are expected on the way out; don't
|
||||
|
||||
@@ -145,6 +145,19 @@ export function computeCorpusGeneration(args: {
|
||||
return h.digest('hex').slice(0, 16);
|
||||
}
|
||||
|
||||
/**
|
||||
* #3507 — the corpus_generation a page lands on when a plain re-embed path
|
||||
* (`embed --stale` and friends) re-embeds a `per_chunk_synopsis` page at the
|
||||
* title-only tier (the D14 fallback tier; synopsis re-generation is a paid
|
||||
* backfill concern). Callers restamp
|
||||
* `updatePageContextualRetrievalState(slug, sourceId, 'title', titleTierCorpusGeneration())`
|
||||
* so the stamped mode keeps describing the vectors actually in the column.
|
||||
* Matches what the inline import path writes for its title-tier pages.
|
||||
*/
|
||||
export function titleTierCorpusGeneration(): string {
|
||||
return computeCorpusGeneration({ crMode: 'title', haikuModel: DEFAULT_HAIKU_MODEL });
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute source_text_hash for D27 P1-4 cache key composition. The
|
||||
* synopsis cache invalidates correctly when adjacent text changes (page
|
||||
|
||||
+17
-2
@@ -19,7 +19,8 @@
|
||||
|
||||
import type { BrainEngine } from './engine.ts';
|
||||
import type { ChunkInput } from './types.ts';
|
||||
import { embedBatchWithBackoff } from '../commands/embed.ts';
|
||||
import { embedBatchWithBackoff, restampIfDemotedToTitleTier } from '../commands/embed.ts';
|
||||
import { wrapChunkTextsForStoredMode } from './embedding-context.ts';
|
||||
import { type DbPacer, createNoopPacer, observed } from './db-pacer.ts';
|
||||
import { AbortError } from './abort-check.ts';
|
||||
|
||||
@@ -189,8 +190,15 @@ export async function embedStaleForSource(
|
||||
const keySourceId = stale[0]?.source_id ?? sourceId;
|
||||
const slug = stale[0].slug;
|
||||
try {
|
||||
// #3507: fetch the page row for its title + stored CR mode so the
|
||||
// re-embed reproduces the page's wrapping convention instead of
|
||||
// silently stripping contextual prefixes (mirrors
|
||||
// src/commands/embed.ts:embedAllStale).
|
||||
const pageRow = await observed(pacer, () =>
|
||||
engine.getPage(slug, { sourceId: keySourceId }),
|
||||
);
|
||||
const embeddings = await embedFn(
|
||||
stale.map((c) => c.chunk_text),
|
||||
wrapChunkTextsForStoredMode(pageRow, stale),
|
||||
{ abortSignal: signal },
|
||||
);
|
||||
const existing = await observed(pacer, () =>
|
||||
@@ -233,6 +241,13 @@ export async function embedStaleForSource(
|
||||
engine.setPageEmbeddingSignature(slug, { sourceId: keySourceId, signature }),
|
||||
);
|
||||
}
|
||||
// #3507: a FULLY re-embedded per_chunk_synopsis page landed at the
|
||||
// title tier — keep the stamped mode honest (mixed pages stay as-is).
|
||||
if (stale.length === existing.length) {
|
||||
await observed(pacer, () =>
|
||||
restampIfDemotedToTitleTier(engine, pageRow, slug, keySourceId),
|
||||
);
|
||||
}
|
||||
result.embedded += stale.length;
|
||||
result.pagesProcessed += 1;
|
||||
} catch (e: unknown) {
|
||||
|
||||
@@ -186,3 +186,41 @@ export function modeRequiresHaiku(mode: CRMode): boolean {
|
||||
export function modeRequiresWrapper(mode: CRMode): boolean {
|
||||
return mode !== 'none';
|
||||
}
|
||||
|
||||
/**
|
||||
* #3507 — build the embedding inputs for a re-embed of EXISTING chunk rows,
|
||||
* reproducing the wrapping convention the page's vectors were originally
|
||||
* built under (recorded in `pages.contextual_retrieval_mode`).
|
||||
*
|
||||
* Used by every plain re-embed path (`embed <slug>`, `embed --all`,
|
||||
* `embed --stale`, the embed-backfill Minion loop). Before this helper those
|
||||
* paths embedded raw `chunk_text`, so any re-embed — including the NORMAL
|
||||
* post-model-migration `embed --stale` — silently replaced context-wrapped
|
||||
* vectors with unwrapped ones, degrading retrieval with no signature change
|
||||
* to show for it.
|
||||
*
|
||||
* Convention rules (embed PRESERVES conventions; changing them is
|
||||
* sync/reindex's job):
|
||||
* - mode NULL/undefined/'none' → raw chunk_text (status quo).
|
||||
* - mode 'title' → title-only prefix (pure string concat).
|
||||
* - mode 'per_chunk_synopsis' → title-only prefix. Re-generating Haiku
|
||||
* synopses is a paid backfill concern; title-only is the service's own
|
||||
* documented fallback tier (D14). Callers that fully re-embed a page
|
||||
* this way should restamp the page to 'title' so the column stays
|
||||
* honest (see contextual-retrieval-service.ts:titleTierCorpusGeneration).
|
||||
* - `fenced_code` chunks are NEVER wrapped (D20-T4), same as sync.
|
||||
*/
|
||||
export function wrapChunkTextsForStoredMode(
|
||||
page:
|
||||
| { title?: string | null; contextual_retrieval_mode?: CRMode | null }
|
||||
| null
|
||||
| undefined,
|
||||
chunks: ReadonlyArray<{ chunk_text: string; chunk_source?: string | null }>,
|
||||
): string[] {
|
||||
const mode = page?.contextual_retrieval_mode;
|
||||
if (mode == null || !modeRequiresWrapper(mode)) {
|
||||
return chunks.map((c) => c.chunk_text);
|
||||
}
|
||||
const prefix = buildContextualPrefix(page?.title ?? '', null);
|
||||
return chunks.map((c) => wrapChunkForEmbedding(c.chunk_text, prefix, c.chunk_source));
|
||||
}
|
||||
|
||||
@@ -420,8 +420,10 @@ export class PGLiteEngine implements BrainEngine {
|
||||
let model: string = DEFAULT_EMBEDDING_MODEL;
|
||||
try {
|
||||
const gw = await import('./ai/gateway.ts');
|
||||
// Both accessors THROW when the gateway is unconfigured (they never
|
||||
// return falsy), so the catch below is the only fallback path (#3461).
|
||||
dims = gw.getEmbeddingDimensions();
|
||||
model = gw.getEmbeddingModel() || model;
|
||||
model = gw.getEmbeddingModel();
|
||||
} catch { /* gateway not configured — use defaults */ }
|
||||
|
||||
await this.db.exec(getPGLiteSchema(dims, model));
|
||||
@@ -979,7 +981,8 @@ export class PGLiteEngine implements BrainEngine {
|
||||
const { rows } = await this.db.query(
|
||||
`SELECT id, source_id, slug, type, title, compiled_truth, timeline, frontmatter, content_hash, created_at, updated_at, deleted_at,
|
||||
effective_date, effective_date_source,
|
||||
source_kind, source_uri, ingested_via, ingested_at
|
||||
source_kind, source_uri, ingested_via, ingested_at,
|
||||
contextual_retrieval_mode
|
||||
FROM pages WHERE ${where.join(' AND ')} LIMIT 1`,
|
||||
params
|
||||
);
|
||||
@@ -2320,15 +2323,26 @@ export class PGLiteEngine implements BrainEngine {
|
||||
|
||||
// Provenance fallback for chunks without an explicit `model`: resolve the
|
||||
// gateway's runtime model, not the compile-time DEFAULT_EMBEDDING_MODEL.
|
||||
// See postgres-engine.ts _upsertChunksOnce for the full rationale — pglite
|
||||
// mirrors it for parity.
|
||||
let resolvedModel: string = DEFAULT_EMBEDDING_MODEL;
|
||||
// #3461: getEmbeddingModel() THROWS when unconfigured (never returns
|
||||
// falsy) — on the throw path fall back to the brain's own
|
||||
// `config.embedding_model` row, then the compile-time default as the
|
||||
// last resort. See postgres-engine.ts _upsertChunksOnce for the full
|
||||
// rationale — pglite mirrors it for parity.
|
||||
let resolvedModel: string | null = null;
|
||||
try {
|
||||
const gw = await import('./ai/gateway.ts');
|
||||
resolvedModel = gw.getEmbeddingModel() || resolvedModel;
|
||||
resolvedModel = gw.getEmbeddingModel();
|
||||
} catch {
|
||||
// Gateway unconfigured (unit tests / pre-connect): keep the default.
|
||||
try {
|
||||
const cfg = await this.db.query(
|
||||
`SELECT value FROM config WHERE key = 'embedding_model'`,
|
||||
);
|
||||
resolvedModel = ((cfg.rows[0] as { value?: string } | undefined)?.value) ?? null;
|
||||
} catch {
|
||||
// config table unreadable — fall through to the compile-time default.
|
||||
}
|
||||
}
|
||||
if (!resolvedModel) resolvedModel = DEFAULT_EMBEDDING_MODEL;
|
||||
|
||||
for (const chunk of chunks) {
|
||||
const embeddingStr = chunk.embedding
|
||||
@@ -2381,6 +2395,9 @@ export class PGLiteEngine implements BrainEngine {
|
||||
// Code-chunk metadata columns follow the same chunk_text-gated CASE pattern as `embedding`
|
||||
// (#769). Re-chunk trusts EXCLUDED outright; pure re-embed COALESCEs so a caller carrying
|
||||
// only embedding-shaped fields doesn't clobber metadata to NULL.
|
||||
//
|
||||
// #3461: `model` mirrors the `embedding` CASE branch-for-branch so the label always
|
||||
// describes whichever vector wins the upsert. See postgres-engine.ts for rationale.
|
||||
await this.db.query(
|
||||
`INSERT INTO content_chunks ${cols} VALUES ${rowParts.join(', ')}
|
||||
ON CONFLICT (page_id, chunk_index) DO UPDATE SET
|
||||
@@ -2394,7 +2411,14 @@ export class PGLiteEngine implements BrainEngine {
|
||||
THEN EXCLUDED.embedding
|
||||
ELSE content_chunks.embedding
|
||||
END,
|
||||
model = COALESCE(EXCLUDED.model, content_chunks.model),
|
||||
model = CASE
|
||||
WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.model
|
||||
WHEN content_chunks.embedding IS NULL THEN EXCLUDED.model
|
||||
WHEN EXCLUDED.embedded_at IS NOT NULL
|
||||
AND (content_chunks.embedded_at IS NULL OR EXCLUDED.embedded_at > content_chunks.embedded_at)
|
||||
THEN EXCLUDED.model
|
||||
ELSE content_chunks.model
|
||||
END,
|
||||
token_count = EXCLUDED.token_count,
|
||||
embedded_at = CASE
|
||||
WHEN EXCLUDED.chunk_text != content_chunks.chunk_text AND EXCLUDED.embedding IS NULL THEN NULL
|
||||
|
||||
@@ -382,8 +382,10 @@ export class PostgresEngine implements BrainEngine {
|
||||
let model: string = DEFAULT_EMBEDDING_MODEL;
|
||||
try {
|
||||
const gw = await import('./ai/gateway.ts');
|
||||
// Both accessors THROW when the gateway is unconfigured (they never
|
||||
// return falsy), so the catch below is the only fallback path (#3461).
|
||||
dims = gw.getEmbeddingDimensions();
|
||||
model = gw.getEmbeddingModel() || model;
|
||||
model = gw.getEmbeddingModel();
|
||||
} catch { /* gateway not yet configured — use defaults */ }
|
||||
|
||||
const sqlText = getPostgresSchema(dims, model);
|
||||
@@ -1031,7 +1033,8 @@ export class PostgresEngine implements BrainEngine {
|
||||
const rows = await tx`
|
||||
SELECT id, source_id, slug, type, title, compiled_truth, timeline, frontmatter, content_hash, created_at, updated_at, deleted_at,
|
||||
effective_date, effective_date_source,
|
||||
source_kind, source_uri, ingested_via, ingested_at
|
||||
source_kind, source_uri, ingested_via, ingested_at,
|
||||
contextual_retrieval_mode
|
||||
FROM pages
|
||||
WHERE slug = ${slug} ${sourceCondition} ${deletedCondition}
|
||||
LIMIT 1
|
||||
@@ -2437,14 +2440,28 @@ export class PostgresEngine implements BrainEngine {
|
||||
// hardcoded default (e.g. zeroentropyai:zembed-1) onto rows whose vectors
|
||||
// were produced by a different, config-resolved model — corrupting the
|
||||
// provenance that signature-drift staleness + dim-migration logic trust.
|
||||
// Mirrors the resolve-then-fallback pattern used for schema sizing above.
|
||||
let resolvedModel: string = DEFAULT_EMBEDDING_MODEL;
|
||||
//
|
||||
// #3461: getEmbeddingModel() THROWS when the gateway is unconfigured —
|
||||
// it never returns falsy — so an `||` guard here is dead code and the
|
||||
// catch path used to stamp the compile-time default onto rows whose
|
||||
// vectors came from the config-resolved provider. On the throw path we
|
||||
// now fall back to the brain's own `config.embedding_model` row (kept
|
||||
// current by init / migrate / retrieval-upgrade), which names the model
|
||||
// that actually produced this brain's vectors. The compile-time default
|
||||
// is the LAST resort (fresh brain whose config row doesn't exist yet).
|
||||
let resolvedModel: string | null = null;
|
||||
try {
|
||||
const gw = await import('./ai/gateway.ts');
|
||||
resolvedModel = gw.getEmbeddingModel() || resolvedModel;
|
||||
resolvedModel = gw.getEmbeddingModel();
|
||||
} catch {
|
||||
// Gateway unconfigured (unit tests / pre-connect): keep the default.
|
||||
try {
|
||||
const cfg = await sql`SELECT value FROM config WHERE key = 'embedding_model'`;
|
||||
resolvedModel = (cfg[0]?.value as string | undefined) ?? null;
|
||||
} catch {
|
||||
// config table unreadable — fall through to the compile-time default.
|
||||
}
|
||||
}
|
||||
if (!resolvedModel) resolvedModel = DEFAULT_EMBEDDING_MODEL;
|
||||
|
||||
for (const chunk of chunks) {
|
||||
const embeddingStr = chunk.embedding
|
||||
@@ -2508,6 +2525,11 @@ export class PostgresEngine implements BrainEngine {
|
||||
// pure re-embed (chunk_text unchanged) COALESCEs so a caller that only carries embedding
|
||||
// doesn't clobber metadata to NULL. Without this, every embed --stale pass nuked code-def's
|
||||
// primary index for thousands of chunks at once.
|
||||
//
|
||||
// #3461: `model` mirrors the `embedding` CASE branch-for-branch — the label must
|
||||
// describe whichever vector WINS the upsert. The old COALESCE(EXCLUDED.model, …)
|
||||
// relabeled preserved (older-model) vectors with the current gateway model on every
|
||||
// partial re-embed, corrupting provenance without changing the vector.
|
||||
await sql.unsafe(
|
||||
`INSERT INTO content_chunks ${cols} VALUES ${rows.join(', ')}
|
||||
ON CONFLICT (page_id, chunk_index) DO UPDATE SET
|
||||
@@ -2521,7 +2543,14 @@ export class PostgresEngine implements BrainEngine {
|
||||
THEN EXCLUDED.embedding
|
||||
ELSE content_chunks.embedding
|
||||
END,
|
||||
model = COALESCE(EXCLUDED.model, content_chunks.model),
|
||||
model = CASE
|
||||
WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.model
|
||||
WHEN content_chunks.embedding IS NULL THEN EXCLUDED.model
|
||||
WHEN EXCLUDED.embedded_at IS NOT NULL
|
||||
AND (content_chunks.embedded_at IS NULL OR EXCLUDED.embedded_at > content_chunks.embedded_at)
|
||||
THEN EXCLUDED.model
|
||||
ELSE content_chunks.model
|
||||
END,
|
||||
token_count = EXCLUDED.token_count,
|
||||
embedded_at = CASE
|
||||
WHEN EXCLUDED.chunk_text != content_chunks.chunk_text AND EXCLUDED.embedding IS NULL THEN NULL
|
||||
|
||||
@@ -110,6 +110,12 @@ export function rowToPage(row: Record<string, unknown>): Page {
|
||||
const sourceUri = row.source_uri === undefined ? undefined : (row.source_uri as string | null);
|
||||
const ingestedVia = row.ingested_via === undefined ? undefined : (row.ingested_via as string | null);
|
||||
const ingestedAt = readOptionalDate(row.ingested_at);
|
||||
// #3507: the CR tier the page was last embedded under (three-state, same
|
||||
// pattern as the provenance columns above). Re-embed paths (`embed --stale`
|
||||
// and friends) read this to reproduce the page's stored wrapping convention.
|
||||
const contextualRetrievalMode = row.contextual_retrieval_mode === undefined
|
||||
? undefined
|
||||
: (row.contextual_retrieval_mode as Page['contextual_retrieval_mode']);
|
||||
return {
|
||||
id: row.id as number,
|
||||
slug: row.slug as string,
|
||||
@@ -135,6 +141,7 @@ export function rowToPage(row: Record<string, unknown>): Page {
|
||||
...(sourceUri !== undefined && { source_uri: sourceUri }),
|
||||
...(ingestedVia !== undefined && { ingested_via: ingestedVia }),
|
||||
...(ingestedAt !== undefined && { ingested_at: ingestedAt }),
|
||||
...(contextualRetrievalMode !== undefined && { contextual_retrieval_mode: contextualRetrievalMode }),
|
||||
// v0.31.12: propagate source_id so downstream callers (embed, reconcile-links)
|
||||
// can thread it through getChunks / upsertChunks without defaulting to 'default'.
|
||||
// v0.32.8: Page.source_id is required. Every SELECT feeding rowToPage now
|
||||
|
||||
@@ -252,6 +252,87 @@ describe('upsertChunks — model provenance uses gateway-resolved model, not com
|
||||
|
||||
resetGateway();
|
||||
});
|
||||
|
||||
// #3461: getEmbeddingModel() THROWS when the gateway is unconfigured — it
|
||||
// never returns falsy — so the reland's `|| resolvedModel` guard was dead
|
||||
// code and the catch path still stamped the compile-time default onto rows
|
||||
// whose vectors came from the config-resolved provider. The engine must
|
||||
// fall back to the brain's own `config.embedding_model` row instead.
|
||||
test('#3461: unconfigured gateway falls back to the brain config model, never the compiled default', async () => {
|
||||
await engine.setConfig('embedding_model', 'voyage:voyage-3-large');
|
||||
// The preload's beforeEach re-configures the gateway before every test,
|
||||
// so the reset must happen INSIDE the test body.
|
||||
resetGateway();
|
||||
|
||||
await engine.putPage('docs/provenance-throw-path', {
|
||||
type: 'concept',
|
||||
title: 'Provenance throw-path page',
|
||||
compiled_truth: 'Chunk written while the gateway is unconfigured.',
|
||||
});
|
||||
await engine.upsertChunks('docs/provenance-throw-path', [
|
||||
{ chunk_index: 0, chunk_text: 'throw-path provenance chunk', chunk_source: 'compiled_truth' },
|
||||
]);
|
||||
|
||||
const rows = await engine.executeRaw<{ model: string }>(
|
||||
`SELECT cc.model FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
WHERE p.slug = 'docs/provenance-throw-path'`,
|
||||
);
|
||||
expect(rows.length).toBe(1);
|
||||
expect(rows[0].model).toBe('voyage:voyage-3-large');
|
||||
|
||||
// Restore the value initSchema wrote for the rest of the file.
|
||||
await engine.setConfig('embedding_model', 'openai:text-embedding-3-large');
|
||||
});
|
||||
|
||||
// #3461 sibling: on a partial re-upsert that carries NO new embedding (the
|
||||
// exact shape `embed --stale` produces for a page's non-stale chunks), the
|
||||
// preserved vector must KEEP its original model label. The old
|
||||
// COALESCE(EXCLUDED.model, …) relabeled it with the current gateway model.
|
||||
test('#3461: preserved vector keeps its original model label on a no-embedding re-upsert', async () => {
|
||||
configureGateway({
|
||||
embedding_model: 'openai:text-embedding-3-large',
|
||||
embedding_dimensions: 1536,
|
||||
env: { OPENAI_API_KEY: 'sk-test' },
|
||||
});
|
||||
|
||||
await engine.putPage('docs/provenance-preserve', {
|
||||
type: 'concept',
|
||||
title: 'Provenance preserve page',
|
||||
compiled_truth: 'Chunk embedded under model A, re-upserted under model B.',
|
||||
});
|
||||
await engine.upsertChunks('docs/provenance-preserve', [
|
||||
{
|
||||
chunk_index: 0,
|
||||
chunk_text: 'stable chunk text',
|
||||
chunk_source: 'compiled_truth',
|
||||
embedding: new Float32Array(VEC1536_A),
|
||||
},
|
||||
]);
|
||||
|
||||
// Model swap: the gateway now resolves a different model, and the
|
||||
// re-upsert (same chunk_text) carries no new embedding.
|
||||
configureGateway({
|
||||
embedding_model: 'voyage:voyage-3-large',
|
||||
embedding_dimensions: 1536,
|
||||
env: { VOYAGE_API_KEY: 'test' },
|
||||
});
|
||||
await engine.upsertChunks('docs/provenance-preserve', [
|
||||
{ chunk_index: 0, chunk_text: 'stable chunk text', chunk_source: 'compiled_truth' },
|
||||
]);
|
||||
|
||||
const rows = await engine.executeRaw<{ model: string; has_embedding: boolean }>(
|
||||
`SELECT cc.model, cc.embedding IS NOT NULL AS has_embedding
|
||||
FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
WHERE p.slug = 'docs/provenance-preserve'`,
|
||||
);
|
||||
expect(rows.length).toBe(1);
|
||||
expect(rows[0].has_embedding).toBe(true); // vector preserved…
|
||||
expect(rows[0].model).toBe('openai:text-embedding-3-large'); // …and its label still describes it
|
||||
|
||||
resetGateway();
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildVectorCastFragment — engine SQL composer (D3)', () => {
|
||||
|
||||
@@ -277,3 +277,79 @@ describe('embedStaleForSource', () => {
|
||||
expect(txtRow.embedded_at).not.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// ────────────────────────────────────────────────────────────────
|
||||
// #3507 — re-embed must reproduce the page's STORED contextual-retrieval
|
||||
// wrapping convention. Before the fix, every plain re-embed (including the
|
||||
// normal post-model-migration `embed --stale`) embedded raw chunk_text,
|
||||
// silently replacing context-wrapped vectors with unwrapped ones.
|
||||
// ────────────────────────────────────────────────────────────────
|
||||
|
||||
describe('contextual-retrieval wrapping on re-embed (#3507)', () => {
|
||||
/** embedFn that records every text it is asked to embed. */
|
||||
function capturingEmbedFn(seen: string[]) {
|
||||
return (texts: string[]): Promise<Float32Array[]> => {
|
||||
seen.push(...texts);
|
||||
return fakeEmbedFn(texts);
|
||||
};
|
||||
}
|
||||
|
||||
async function seedWrappablePage(slug: string, title: string): Promise<void> {
|
||||
await engine.putPage(slug, { type: 'note', title, compiled_truth: 'seeded' });
|
||||
await engine.upsertChunks(slug, [
|
||||
{ chunk_index: 0, chunk_text: 'prose chunk about widgets', chunk_source: 'compiled_truth', token_count: 4 },
|
||||
{ chunk_index: 1, chunk_text: 'const x = 1;', chunk_source: 'fenced_code', token_count: 4 },
|
||||
]);
|
||||
}
|
||||
|
||||
test('title-mode page: stale re-embed sends title-wrapped texts; fenced_code stays raw', async () => {
|
||||
await seedWrappablePage('wrapped-page', 'Widget Notes');
|
||||
await engine.updatePageContextualRetrievalState('wrapped-page', 'default', 'title', 'gen-title');
|
||||
|
||||
const seen: string[] = [];
|
||||
const result = await embedStaleForSource(engine, 'default', { embedFn: capturingEmbedFn(seen) });
|
||||
expect(result.embedded).toBe(2);
|
||||
|
||||
expect(seen).toContain('<context>Widget Notes\n</context>\nprose chunk about widgets');
|
||||
expect(seen).toContain('const x = 1;'); // fenced_code is NEVER wrapped (D20-T4)
|
||||
|
||||
// D20-T1: the canonical chunk_text is NOT rewritten — wrapping is embed-input-only.
|
||||
const chunks = await engine.getChunks('wrapped-page');
|
||||
expect(chunks.map((c) => c.chunk_text).sort()).toEqual(['const x = 1;', 'prose chunk about widgets']);
|
||||
// Mode stamp unchanged for title-tier pages.
|
||||
const rows = await engine.executeRaw<{ contextual_retrieval_mode: string }>(
|
||||
`SELECT contextual_retrieval_mode FROM pages WHERE slug = 'wrapped-page'`,
|
||||
);
|
||||
expect(rows[0].contextual_retrieval_mode).toBe('title');
|
||||
});
|
||||
|
||||
test('per_chunk_synopsis page: re-embed applies the title-tier wrapper and restamps honestly', async () => {
|
||||
await seedWrappablePage('synopsis-page', 'Synopsis Notes');
|
||||
await engine.updatePageContextualRetrievalState('synopsis-page', 'default', 'per_chunk_synopsis', 'gen-synopsis');
|
||||
|
||||
const seen: string[] = [];
|
||||
const result = await embedStaleForSource(engine, 'default', { embedFn: capturingEmbedFn(seen) });
|
||||
expect(result.embedded).toBe(2);
|
||||
|
||||
// Synopsis re-generation is a paid backfill concern; the plain re-embed
|
||||
// lands at the title tier (the service's own D14 fallback tier)…
|
||||
expect(seen).toContain('<context>Synopsis Notes\n</context>\nprose chunk about widgets');
|
||||
// …and the stamped mode is updated so it keeps describing the vectors.
|
||||
const rows = await engine.executeRaw<{ contextual_retrieval_mode: string }>(
|
||||
`SELECT contextual_retrieval_mode FROM pages WHERE slug = 'synopsis-page'`,
|
||||
);
|
||||
expect(rows[0].contextual_retrieval_mode).toBe('title');
|
||||
});
|
||||
|
||||
test('unstamped page (NULL mode) embeds raw chunk_text — convention preserved', async () => {
|
||||
await seedWrappablePage('plain-page', 'Plain Notes');
|
||||
// No updatePageContextualRetrievalState call: pre-CR page.
|
||||
|
||||
const seen: string[] = [];
|
||||
const result = await embedStaleForSource(engine, 'default', { embedFn: capturingEmbedFn(seen) });
|
||||
expect(result.embedded).toBe(2);
|
||||
|
||||
expect(seen).toContain('prose chunk about widgets');
|
||||
expect(seen.some((t) => t.startsWith('<context>'))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -907,3 +907,76 @@ describe('runEmbed preserves code-chunk metadata across re-embed (regression for
|
||||
expect(metadataOf(upsertChunkArgs![0])).toEqual(metadataOf(fullCodeChunk));
|
||||
});
|
||||
});
|
||||
|
||||
// ────────────────────────────────────────────────────────────────
|
||||
// #3507 — `embed --stale` must reproduce the page's STORED
|
||||
// contextual-retrieval wrapping convention instead of embedding raw
|
||||
// chunk_text (which silently stripped contextual prefixes on every
|
||||
// re-embed, including the normal post-model-migration path).
|
||||
// ────────────────────────────────────────────────────────────────
|
||||
|
||||
describe('embed --stale contextual-retrieval wrapping (#3507)', () => {
|
||||
const wrapChunks = [
|
||||
{ chunk_index: 0, chunk_text: 'prose chunk', chunk_source: 'compiled_truth', embedded_at: null, token_count: 1 },
|
||||
{ chunk_index: 1, chunk_text: 'const x = 1;', chunk_source: 'fenced_code', embedded_at: null, token_count: 1 },
|
||||
];
|
||||
const wrapStale = [
|
||||
{ slug: 'wrapped', chunk_index: 0, chunk_text: 'prose chunk', chunk_source: 'compiled_truth' as const, model: null, token_count: 1, source_id: 'default', page_id: 1 },
|
||||
{ slug: 'wrapped', chunk_index: 1, chunk_text: 'const x = 1;', chunk_source: 'fenced_code' as any, model: null, token_count: 1, source_id: 'default', page_id: 1 },
|
||||
];
|
||||
|
||||
function wrappingHarness(mode: string | null) {
|
||||
const seen: string[] = [];
|
||||
const restamps: any[][] = [];
|
||||
embedBatchBehavior = async (texts: string[]) => {
|
||||
seen.push(...texts);
|
||||
return texts.map(() => new Float32Array(1536));
|
||||
};
|
||||
const engine = mockEngine({
|
||||
countStaleChunks: async () => 2,
|
||||
listStaleChunks: async () => wrapStale,
|
||||
getPage: async () => ({
|
||||
slug: 'wrapped',
|
||||
title: 'Widget Notes',
|
||||
source_id: 'default',
|
||||
compiled_truth: 'x',
|
||||
timeline: '',
|
||||
contextual_retrieval_mode: mode,
|
||||
}),
|
||||
getChunks: async () => wrapChunks,
|
||||
upsertChunks: async () => {},
|
||||
updatePageContextualRetrievalState: async (...args: any[]) => { restamps.push(args); },
|
||||
});
|
||||
return { engine, seen, restamps };
|
||||
}
|
||||
|
||||
test('title-mode page: stale re-embed wraps prose with the title prefix; fenced_code stays raw', async () => {
|
||||
const { engine, seen, restamps } = wrappingHarness('title');
|
||||
const result = await runEmbedCore(engine, { stale: true });
|
||||
expect(result.embedded).toBe(2);
|
||||
expect(seen).toContain('<context>Widget Notes\n</context>\nprose chunk');
|
||||
expect(seen).toContain('const x = 1;');
|
||||
expect(restamps).toHaveLength(0); // title tier: stamp already honest
|
||||
});
|
||||
|
||||
test('per_chunk_synopsis page: fully re-embedded page restamps to the title tier', async () => {
|
||||
const { engine, seen, restamps } = wrappingHarness('per_chunk_synopsis');
|
||||
const result = await runEmbedCore(engine, { stale: true });
|
||||
expect(result.embedded).toBe(2);
|
||||
expect(seen).toContain('<context>Widget Notes\n</context>\nprose chunk');
|
||||
expect(restamps).toHaveLength(1);
|
||||
const [slug, sourceId, newMode] = restamps[0];
|
||||
expect(slug).toBe('wrapped');
|
||||
expect(sourceId).toBe('default');
|
||||
expect(newMode).toBe('title');
|
||||
});
|
||||
|
||||
test('page with no stored CR mode embeds raw chunk_text (convention preserved)', async () => {
|
||||
const { engine, seen, restamps } = wrappingHarness(null);
|
||||
const result = await runEmbedCore(engine, { stale: true });
|
||||
expect(result.embedded).toBe(2);
|
||||
expect(seen).toContain('prose chunk');
|
||||
expect(seen.some((t) => t.startsWith('<context>'))).toBe(false);
|
||||
expect(restamps).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user