diff --git a/CHANGELOG.md b/CHANGELOG.md index 170c872d5..ccb582eb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ All notable changes to GBrain will be documented in this file. +## [0.22.6] - 2026-04-28 + +### Schema verification after migrations + +- Post-migration schema verification catches columns that were defined in migrations but silently failed to create (common with PgBouncer transaction-mode poolers). +- Self-healing: automatically adds missing columns via ALTER TABLE when detected. +- Prevents the "column X does not exist" embed failures that occur when schema version is ahead of actual table state. + ## [0.22.5] - 2026-04-27 ## **Autopilot stops re-importing your whole brain when a commit gets garbage-collected.** diff --git a/package.json b/package.json index fb31c277a..4b974e4be 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gbrain", - "version": "0.22.5", + "version": "0.22.6", "description": "Postgres-native personal knowledge brain with hybrid RAG search", "type": "module", "main": "src/core/index.ts", diff --git a/src/core/db.ts b/src/core/db.ts index 10f18175e..531f0d316 100644 --- a/src/core/db.ts +++ b/src/core/db.ts @@ -2,6 +2,7 @@ import postgres from 'postgres'; import { GBrainError, type EngineConfig } from './types.ts'; import { SCHEMA_SQL } from './schema-embedded.ts'; import type { BrainEngine } from './engine.ts'; +import { verifySchema } from './schema-verify.ts'; let sql: ReturnType | null = null; let connectedUrl: string | null = null; @@ -237,6 +238,8 @@ export async function initSchema(): Promise { } } +export { verifySchema } from './schema-verify.ts'; + export async function withTransaction(fn: (tx: ReturnType) => Promise): Promise { const conn = getConnection(); return conn.begin(async (tx) => { diff --git a/src/core/postgres-engine.ts b/src/core/postgres-engine.ts index 865e9a63e..7ecd0d925 100644 --- a/src/core/postgres-engine.ts +++ b/src/core/postgres-engine.ts @@ -3,6 +3,7 @@ import type { BrainEngine, LinkBatchInput, TimelineBatchInput, ReservedConnectio import { MAX_SEARCH_LIMIT, clampSearchLimit } from './engine.ts'; import { runMigrations } from './migrate.ts'; import { SCHEMA_SQL } from './schema-embedded.ts'; +import { verifySchema } from './schema-verify.ts'; import type { Page, PageInput, PageFilters, PageType, Chunk, ChunkInput, StaleChunkRow, @@ -108,6 +109,14 @@ export class PostgresEngine implements BrainEngine { if (applied > 0) { console.log(` ${applied} migration(s) applied`); } + + // Post-migration schema verification: catches columns that migrations + // defined but PgBouncer transaction-mode silently failed to create. + // Self-heals missing columns via ALTER TABLE ADD COLUMN IF NOT EXISTS. + const verify = await verifySchema(this); + if (verify.healed.length > 0) { + console.log(` Schema verify: self-healed ${verify.healed.length} missing column(s)`); + } } finally { await conn`SELECT pg_advisory_unlock(42)`; } diff --git a/src/core/schema-verify.ts b/src/core/schema-verify.ts new file mode 100644 index 000000000..64250e83b --- /dev/null +++ b/src/core/schema-verify.ts @@ -0,0 +1,282 @@ +/** + * Post-migration schema verification with self-healing. + * + * PgBouncer transaction-mode poolers can silently swallow ALTER TABLE + * statements: the SQL doesn't error, but the column never gets created. + * The migration system increments the schema version counter anyway, so + * gbrain thinks it's on v29 but the actual table is missing columns. + * + * This module parses the canonical CREATE TABLE definitions in + * schema-embedded.ts and diffs them against information_schema.columns. + * Missing columns are self-healed via ALTER TABLE ADD COLUMN IF NOT EXISTS. + * + * Called at the end of initSchema(), after all migrations complete. + */ + +import { SCHEMA_SQL } from './schema-embedded.ts'; +import type { BrainEngine } from './engine.ts'; + +/** A column expected to exist in the database. */ +export interface ExpectedColumn { + table: string; + column: string; + /** The full column definition (type + constraints) from the CREATE TABLE. */ + definition: string; +} + +/** + * Parse CREATE TABLE statements from SCHEMA_SQL to extract expected columns. + * + * This is a best-effort parser that handles the gbrain schema conventions: + * - Standard column definitions with types and constraints + * - Skips CONSTRAINT lines, CHECK lines, and UNIQUE lines + * - Handles multi-line definitions + * + * Returns only tables and columns — not constraints, indexes, or triggers. + */ +export function parseExpectedColumns(): ExpectedColumn[] { + const results: ExpectedColumn[] = []; + + // Match CREATE TABLE IF NOT EXISTS ( ... ); + const tableRegex = /CREATE\s+TABLE\s+IF\s+NOT\s+EXISTS\s+(\w+)\s*\(([\s\S]*?)\);/gi; + + const SQL_KEYWORDS = new Set(['constraint', 'unique', 'check', 'primary', 'foreign', 'exclude']); + + function processLine(tableName: string, line: string) { + line = line.trim().replace(/,\s*$/, ''); + if (!line) return; + + // Skip CONSTRAINT, UNIQUE, CHECK, PRIMARY KEY lines + if (/^\s*(CONSTRAINT|UNIQUE|CHECK|PRIMARY\s+KEY)/i.test(line)) return; + + const colMatch = line.match(/^\s*(\w+)\s+(.+)$/); + if (colMatch) { + const colName = colMatch[1].toLowerCase(); + if (SQL_KEYWORDS.has(colName)) return; + + results.push({ + table: tableName, + column: colName, + definition: colMatch[2].trim(), + }); + } + } + + let match: RegExpExecArray | null; + while ((match = tableRegex.exec(SCHEMA_SQL)) !== null) { + const tableName = match[1]; + const body = match[2]; + + const lines = body.split('\n'); + let currentLine = ''; + + for (const rawLine of lines) { + const trimmed = rawLine.trim(); + + // Skip empty lines and comments + if (!trimmed || trimmed.startsWith('--')) { + // If we have accumulated content and hit a blank/comment line, + // the accumulated content is a complete line + if (currentLine.trim()) { + processLine(tableName, currentLine); + currentLine = ''; + } + continue; + } + + currentLine += ' ' + trimmed; + + // If line ends with comma, it's a complete column definition + if (trimmed.endsWith(',')) { + processLine(tableName, currentLine); + currentLine = ''; + } + } + + // Handle any remaining accumulated line (last column before closing paren) + if (currentLine.trim()) { + processLine(tableName, currentLine); + } + } + + // Also parse ALTER TABLE ... ADD COLUMN IF NOT EXISTS statements. + // These are used for columns added outside CREATE TABLE blocks + // (e.g., pages.search_vector, files.source_id). + const alterRegex = /ALTER\s+TABLE\s+(\w+)\s+ADD\s+COLUMN\s+IF\s+NOT\s+EXISTS\s+(\w+)\s+([^;,]+)/gi; + let alterMatch: RegExpExecArray | null; + const seen = new Set(results.map(r => `${r.table}.${r.column}`)); + while ((alterMatch = alterRegex.exec(SCHEMA_SQL)) !== null) { + const table = alterMatch[1]; + const column = alterMatch[2].toLowerCase(); + const definition = alterMatch[3].trim().replace(/,\s*$/, ''); + const key = `${table}.${column}`; + if (!seen.has(key)) { + seen.add(key); + results.push({ table, column, definition }); + } + } + + return results; +} + +/** + * Build a simplified type expression suitable for ALTER TABLE ADD COLUMN. + * + * Strips inline REFERENCES, CHECK, UNIQUE, and complex constraints that + * can't be used in ADD COLUMN IF NOT EXISTS. Preserves NOT NULL, DEFAULT, + * and the base type. + */ +export function simplifyColumnDef(definition: string): string { + let def = definition; + + // Remove REFERENCES ... (with optional ON DELETE/UPDATE clauses) + def = def.replace(/REFERENCES\s+\w+\([^)]*\)(\s+ON\s+(DELETE|UPDATE)\s+\w+(\s+\w+)?)*\s*/gi, ''); + + // Remove CHECK constraints (handle nested parens) + def = def.replace(/CHECK\s*\((?:[^()]*|\([^()]*\))*\)/gi, ''); + + // Remove inline UNIQUE + def = def.replace(/\bUNIQUE\b/gi, ''); + + // Remove trailing commas and whitespace + def = def.replace(/,\s*$/, '').trim(); + + // Collapse multiple spaces + def = def.replace(/\s+/g, ' ').trim(); + + return def; +} + +/** + * Query the database for actual columns in the public schema. + * Returns a Set of "table.column" strings for fast lookup. + */ +async function getActualColumns(engine: BrainEngine): Promise> { + const rows = await engine.executeRaw<{ table_name: string; column_name: string }>( + `SELECT table_name, column_name + FROM information_schema.columns + WHERE table_schema = 'public'` + ); + const set = new Set(); + for (const row of rows) { + set.add(`${row.table_name}.${row.column_name}`); + } + return set; +} + +/** + * Get the set of tables that actually exist in the database. + */ +async function getActualTables(engine: BrainEngine): Promise> { + const rows = await engine.executeRaw<{ table_name: string }>( + `SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' AND table_type = 'BASE TABLE'` + ); + return new Set(rows.map(r => r.table_name)); +} + +export interface VerifyResult { + /** Total columns checked */ + checked: number; + /** Columns that were missing */ + missing: Array<{ table: string; column: string }>; + /** Columns successfully self-healed */ + healed: Array<{ table: string; column: string }>; + /** Columns that failed to self-heal */ + failed: Array<{ table: string; column: string; error: string }>; +} + +/** + * Verify that every column defined in schema-embedded.ts actually exists + * in the database. Self-heals missing columns via ALTER TABLE ADD COLUMN. + * + * Should be called after initSchema() + runMigrations() complete. + * + * @returns VerifyResult with details of what was checked and fixed. + * @throws Error if any columns could not be healed (after attempting all). + */ +export async function verifySchema(engine: BrainEngine): Promise { + const expected = parseExpectedColumns(); + const actualColumns = await getActualColumns(engine); + const actualTables = await getActualTables(engine); + + const result: VerifyResult = { + checked: 0, + missing: [], + healed: [], + failed: [], + }; + + // Group expected columns by table for cleaner logging + for (const col of expected) { + // Skip tables that don't exist yet — they'll be created by schema.sql + // on the next initSchema() call. We only verify columns on tables that + // DO exist (the failure mode is: table exists, migration ran, but ALTER + // TABLE silently failed). + if (!actualTables.has(col.table)) { + continue; + } + + result.checked++; + + const key = `${col.table}.${col.column}`; + if (!actualColumns.has(key)) { + result.missing.push({ table: col.table, column: col.column }); + } + } + + if (result.missing.length === 0) { + return result; + } + + // Log missing columns + console.warn(`\n⚠️ Schema verification found ${result.missing.length} missing column(s):`); + for (const m of result.missing) { + console.warn(` ${m.table}.${m.column}`); + } + console.warn(' Attempting self-heal via ALTER TABLE ADD COLUMN...\n'); + + // Build a map from table.column -> definition for self-healing + const defMap = new Map(); + for (const col of expected) { + defMap.set(`${col.table}.${col.column}`, col.definition); + } + + // Attempt to add each missing column + for (const m of result.missing) { + const rawDef = defMap.get(`${m.table}.${m.column}`); + if (!rawDef) { + result.failed.push({ ...m, error: 'No definition found in schema' }); + continue; + } + + const simpleDef = simplifyColumnDef(rawDef); + + try { + const sql = `ALTER TABLE ${m.table} ADD COLUMN IF NOT EXISTS ${m.column} ${simpleDef}`; + await engine.runMigration(0, sql); + result.healed.push({ table: m.table, column: m.column }); + console.log(` ✓ Added ${m.table}.${m.column}`); + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : String(err); + result.failed.push({ ...m, error: msg }); + console.error(` ✗ Failed to add ${m.table}.${m.column}: ${msg}`); + } + } + + if (result.healed.length > 0) { + console.log(`\n Schema self-heal: ${result.healed.length}/${result.missing.length} column(s) recovered.`); + } + + if (result.failed.length > 0) { + const failList = result.failed.map(f => `${f.table}.${f.column}: ${f.error}`).join('\n '); + throw new Error( + `Schema verification failed: ${result.failed.length} column(s) could not be added:\n ${failList}\n` + + 'This usually means PgBouncer transaction-mode silently dropped ALTER TABLE statements.\n' + + 'Fix: connect directly to Postgres (not through PgBouncer) and run: gbrain apply-migrations --yes' + ); + } + + return result; +} diff --git a/test/schema-verify.test.ts b/test/schema-verify.test.ts new file mode 100644 index 000000000..2934e19ae --- /dev/null +++ b/test/schema-verify.test.ts @@ -0,0 +1,108 @@ +import { describe, it, expect } from 'bun:test'; +import { parseExpectedColumns, simplifyColumnDef } from '../src/core/schema-verify.ts'; + +describe('parseExpectedColumns', () => { + it('extracts columns from all major tables', () => { + const columns = parseExpectedColumns(); + + // Should find columns from known tables + const tables = new Set(columns.map(c => c.table)); + expect(tables.has('pages')).toBe(true); + expect(tables.has('content_chunks')).toBe(true); + expect(tables.has('links')).toBe(true); + expect(tables.has('sources')).toBe(true); + expect(tables.has('minion_jobs')).toBe(true); + expect(tables.has('files')).toBe(true); + + // Should find specific columns that have historically been missed by PgBouncer + const columnKeys = new Set(columns.map(c => `${c.table}.${c.column}`)); + expect(columnKeys.has('content_chunks.symbol_type')).toBe(true); + expect(columnKeys.has('content_chunks.start_line')).toBe(true); + expect(columnKeys.has('content_chunks.end_line')).toBe(true); + expect(columnKeys.has('content_chunks.parent_symbol_path')).toBe(true); + expect(columnKeys.has('content_chunks.doc_comment')).toBe(true); + expect(columnKeys.has('content_chunks.symbol_name_qualified')).toBe(true); + expect(columnKeys.has('content_chunks.search_vector')).toBe(true); + + // pages columns + expect(columnKeys.has('pages.slug')).toBe(true); + expect(columnKeys.has('pages.source_id')).toBe(true); + expect(columnKeys.has('pages.page_kind')).toBe(true); + expect(columnKeys.has('pages.search_vector')).toBe(true); + + // sources columns + expect(columnKeys.has('sources.chunker_version')).toBe(true); + }); + + it('does not include CONSTRAINT lines as columns', () => { + const columns = parseExpectedColumns(); + const colNames = columns.map(c => c.column); + + // These are constraint names, not column names + expect(colNames).not.toContain('constraint'); + expect(colNames).not.toContain('unique'); + expect(colNames).not.toContain('check'); + expect(colNames).not.toContain('primary'); + expect(colNames).not.toContain('foreign'); + }); + + it('returns non-empty definitions for all columns', () => { + const columns = parseExpectedColumns(); + for (const col of columns) { + expect(col.definition.length).toBeGreaterThan(0); + } + }); +}); + +describe('simplifyColumnDef', () => { + it('strips REFERENCES clauses', () => { + const result = simplifyColumnDef( + "TEXT NOT NULL DEFAULT 'default' REFERENCES sources(id) ON DELETE CASCADE" + ); + expect(result).toBe("TEXT NOT NULL DEFAULT 'default'"); + }); + + it('strips CHECK constraints', () => { + const result = simplifyColumnDef( + "TEXT NOT NULL DEFAULT 'markdown' CHECK (page_kind IN ('markdown','code'))" + ); + expect(result).toBe("TEXT NOT NULL DEFAULT 'markdown'"); + }); + + it('preserves simple type + NOT NULL + DEFAULT', () => { + const result = simplifyColumnDef("INTEGER NOT NULL DEFAULT 0"); + expect(result).toBe("INTEGER NOT NULL DEFAULT 0"); + }); + + it('strips UNIQUE keyword', () => { + const result = simplifyColumnDef("TEXT NOT NULL UNIQUE"); + expect(result).toBe("TEXT NOT NULL"); + }); + + it('handles complex REFERENCES with ON DELETE and ON UPDATE', () => { + const result = simplifyColumnDef( + "INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE" + ); + expect(result).toBe("INTEGER NOT NULL"); + }); + + it('handles bare type', () => { + const result = simplifyColumnDef("TEXT"); + expect(result).toBe("TEXT"); + }); + + it('handles vector type', () => { + const result = simplifyColumnDef("vector(1536)"); + expect(result).toBe("vector(1536)"); + }); + + it('handles TSVECTOR type', () => { + const result = simplifyColumnDef("TSVECTOR"); + expect(result).toBe("TSVECTOR"); + }); + + it('handles array types', () => { + const result = simplifyColumnDef("TEXT[]"); + expect(result).toBe("TEXT[]"); + }); +});