mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-31 04:07:52 +00:00
Garry TanandClaude Opus 4.7
306fc0e1ef
Adds A4 hard-error path: when `gbrain init --embedding-dimensions N` is run against an existing brain whose `content_chunks.embedding` column is a different `vector(M)`, init exits 1 with an inline four-step ALTER recipe and a pointer to docs/embedding-migrations.md. This kills the silent-corruption pattern surfaced by issue #673: the v0.27 schema seeded `('embedding_dimensions', '1536')` regardless of the flag, so users got a config saying 768 but a column at 1536 — first sync write blew up with "expected 1536, got 768." A4's contract: 1. Connect to engine BEFORE saveConfig so we can read the live column type 2. If column exists AND dim != requested, exit 1 (loud failure) 3. If column doesn't exist (fresh init) OR dim matches, proceed normally Recipe in docs/embedding-migrations.md (and inlined in init's error output) covers all four destructive steps codex's plan-review caught: 1. DROP INDEX IF EXISTS idx_chunks_embedding (HNSW won't survive ALTER) 2. ALTER TABLE content_chunks ALTER COLUMN embedding TYPE vector(N) 3. UPDATE content_chunks SET embedding = NULL, embedded_at = NULL 4. CREATE INDEX HNSW *only if N <= 2000* (pgvector cap) Step 4 is conditional: dims > 2000 (e.g. Voyage 4 Large 2048d) cannot be HNSW-indexed in pgvector; the recipe explicitly says "Skip reindex" in that case so the user doesn't paste a CREATE INDEX that crashes. Helper `readContentChunksEmbeddingDim` and message builder `embeddingMismatchMessage` live in src/core/embedding-dim-check.ts so doctor 8b (next commit) can reuse the same source of truth. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
88 lines
3.5 KiB
TypeScript
88 lines
3.5 KiB
TypeScript
/**
|
|
* v0.28.5 (A4) — Existing-brain dimension-mismatch detection unit tests.
|
|
*
|
|
* Pairs with `gbrain init` and `gbrain doctor`'s loud-failure paths. Validates
|
|
* that:
|
|
* 1. readContentChunksEmbeddingDim correctly reports null on a fresh brain.
|
|
* 2. After initSchema, it returns the actual templated dim (1536 default,
|
|
* or whatever was passed via gateway config).
|
|
* 3. embeddingMismatchMessage produces a recipe that explicitly drops the
|
|
* HNSW index, alters the column, wipes embeddings, and conditionally
|
|
* reindexes — codex's #8 finding from plan review.
|
|
*/
|
|
|
|
import { test, expect, describe } from 'bun:test';
|
|
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
|
import {
|
|
readContentChunksEmbeddingDim,
|
|
embeddingMismatchMessage,
|
|
} from '../src/core/embedding-dim-check.ts';
|
|
|
|
describe('readContentChunksEmbeddingDim', () => {
|
|
test('returns { exists: false, dims: null } on a fresh brain', async () => {
|
|
const engine = new PGLiteEngine();
|
|
await engine.connect({});
|
|
try {
|
|
const result = await readContentChunksEmbeddingDim(engine);
|
|
expect(result.exists).toBe(false);
|
|
expect(result.dims).toBeNull();
|
|
} finally {
|
|
await engine.disconnect();
|
|
}
|
|
}, 30000);
|
|
|
|
test('returns dims from a migrated brain (default 1536)', async () => {
|
|
const engine = new PGLiteEngine();
|
|
await engine.connect({});
|
|
try {
|
|
await engine.initSchema();
|
|
const result = await readContentChunksEmbeddingDim(engine);
|
|
expect(result.exists).toBe(true);
|
|
expect(result.dims).toBe(1536);
|
|
} finally {
|
|
await engine.disconnect();
|
|
}
|
|
}, 30000);
|
|
});
|
|
|
|
describe('embeddingMismatchMessage', () => {
|
|
test('inlines all four recipe steps for HNSW-eligible dims', () => {
|
|
const msg = embeddingMismatchMessage({
|
|
currentDims: 1536,
|
|
requestedDims: 768,
|
|
requestedModel: 'nomic-embed-text',
|
|
source: 'init',
|
|
});
|
|
expect(msg).toContain('vector(1536)');
|
|
expect(msg).toContain('vector(768)');
|
|
expect(msg).toContain('DROP INDEX IF EXISTS idx_chunks_embedding');
|
|
expect(msg).toContain('ALTER TABLE content_chunks ALTER COLUMN embedding TYPE vector(768)');
|
|
expect(msg).toContain('UPDATE content_chunks SET embedding = NULL');
|
|
expect(msg).toContain('CREATE INDEX IF NOT EXISTS idx_chunks_embedding');
|
|
expect(msg).toContain('docs/embedding-migrations.md');
|
|
});
|
|
|
|
test('skips HNSW recreate when requested dims exceed pgvector cap', () => {
|
|
// Codex finding #8: 2048d (Voyage 4 Large) cannot be HNSW-indexed in pgvector.
|
|
// The recipe must NOT instruct a CREATE INDEX HNSW for that dim.
|
|
const msg = embeddingMismatchMessage({
|
|
currentDims: 1536,
|
|
requestedDims: 2048,
|
|
requestedModel: 'voyage-4-large',
|
|
source: 'init',
|
|
});
|
|
expect(msg).toContain('vector(2048)');
|
|
expect(msg).toContain('Skip reindex');
|
|
expect(msg).toContain('exceeds pgvector\'s HNSW cap');
|
|
// The HNSW CREATE INDEX line must NOT appear in the 2048d recipe.
|
|
expect(msg).not.toContain('CREATE INDEX IF NOT EXISTS idx_chunks_embedding\n ON content_chunks USING hnsw');
|
|
});
|
|
|
|
test('source: doctor uses a different header than source: init', () => {
|
|
const initMsg = embeddingMismatchMessage({ currentDims: 1536, requestedDims: 768, source: 'init' });
|
|
const doctorMsg = embeddingMismatchMessage({ currentDims: 1536, requestedDims: 768, source: 'doctor' });
|
|
expect(initMsg).toContain('Refusing to silently re-template');
|
|
expect(doctorMsg).toContain('Embedding dimension mismatch detected');
|
|
});
|
|
});
|