mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-31 04:07:52 +00:00
feat: PGLite engine — local brain, zero infrastructure (v0.7.0) (#41)
* refactor: extract shared utils, add runMigration + getChunksWithEmbeddings to BrainEngine Extract validateSlug, contentHash, rowToPage, rowToChunk, rowToSearchResult from postgres-engine.ts into shared utils.ts. Add rowToChunk includeEmbedding parameter for migration support. Add two new methods to BrainEngine interface: - runMigration(version, sql) — replaces internal eng.sql access in migrate.ts - getChunksWithEmbeddings(slug) — returns chunks with embedding data for migration Replace 'sqlite' with 'pglite' in EngineConfig and GBrainConfig types. Fix loadConfig to infer engine from database_path. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * feat: pluggable engine factory + hybridSearch keyword-only fallback Add createEngine() factory with dynamic imports so PGLite WASM is never loaded for Postgres users. Wire CLI to use factory instead of hardcoded PostgresEngine. Force workers=1 for PGLite imports (single-connection architecture). Fix hybridSearch to check OPENAI_API_KEY before calling embed(). When unset, returns keyword-only results instead of throwing. Critical for local PGLite users who don't need vector search. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * feat: PGLiteEngine — embedded Postgres 17.5 via WASM, same SQL everywhere Full BrainEngine implementation (37 methods) using @electric-sql/pglite. Same SQL as PostgresEngine — tsvector triggers, pgvector HNSW, pg_trgm fuzzy matching, recursive CTEs, JSONB. Only the driver call syntax differs (parameterized queries instead of tagged templates). PGLite schema is the Postgres schema minus RLS, advisory locks, and remote auth tables (access_tokens, mcp_request_log, files). No server. No subscription. One directory. Works offline. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * feat: smart init (PGLite default) + bidirectional engine migration gbrain init now defaults to PGLite — brain ready in 2 seconds, no server needed. Scans target directory: <1000 .md files = PGLite, >=1000 = suggests Supabase. --supabase and --pglite flags override. gbrain migrate --to supabase/pglite transfers all data between engines with manifest-based resume. Copies pages, chunks (with embeddings), tags, timeline, raw data, links, and config. --force overwrites non-empty target. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * test: 60 new tests for PGLite engine, utils, and factory 41 PGLite engine tests covering all 37 BrainEngine methods: CRUD, tsvector keyword search, pg_trgm fuzzy matching, chunk upsert with COALESCE, graph traversal via recursive CTE, transactions, cascade deletes, stats/health, and embedding round-trip. 14 shared utility tests (validateSlug, contentHash, row mappers). 5 engine factory tests (dispatch, error messages). All run in-memory — zero Docker, zero DATABASE_URL, instant in CI. Add P0 TODO: submit Bun PR for WASM embedding in bun build --compile (oven-sh/bun#15032). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * chore: bump version and changelog (v0.7.0) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * docs: update project documentation for v0.7.0 PGLite engine - CLAUDE.md: add PGLite key files, update architecture, add migrate command, add 3 test files - README.md: PGLite as default init, zero-config getting started, migration path to Supabase - docs/ENGINES.md: PGLiteEngine shipped (v0.7), capability matrix, migration docs - docs/SQLITE_ENGINE.md: marked superseded by PGLite Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * chore: remove stale v0.4 README update prompt Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * chore: remove SQLITE_ENGINE.md (superseded by PGLite) PGLite uses the same SQL as Postgres, making a separate SQLite engine unnecessary. docs/ENGINES.md covers PGLiteEngine. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * docs: update README step 2 to default to PGLite Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * docs: add schema setup step and install-all-integrations step to README Step 3 now tells agents to read GBRAIN_RECOMMENDED_SCHEMA.md and set up the MECE directory structure before importing. Step 7 tells agents to install every available integration recipe, not just list them. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * docs: update install goal to match full opinionated setup Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * docs: add 'Need an AI agent first?' section with one-click deploy links New users who don't have OpenClaw or Hermes Agent get pointed to AlphaClaw on Render and the Hermes Agent Railway template. One click each. Claude Code mentioned for users who already have it. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: add migrate to CLI_ONLY + help output, fix standalone example - migrate command was missing from CLI_ONLY set (errored as "Unknown command") - migrate now shows in --help under SETUP - init help line shows --pglite flag - standalone CLI example uses gbrain init (not --supabase) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * docs: set realistic time expectation (~30 min to working brain) DB is 2 seconds. But schema + import + embeddings + integrations is 15-30 minutes. The agent does the work, you answer API key questions. Don't oversell time-to-value. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * docs: fix AlphaClaw Render requirement (8GB+ RAM, not free tier) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * docs: final README polish for launch - GOAL line: "Garry Tan's exact setup" (not Claude Code specific) - Remove markdown links from code block (won't render) - STEP 2 renamed from "START HERE" to "DATABASE" - Tighten Supabase fallback text Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * docs: remove duplicate old install block from README The v0.5-era "With OpenClaw or Hermes Agent" paste block was superseded by the top-level "Start here" block. Having both confused users and the old one still said --supabase as step 2. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * docs: clean up README consistency and remove duplicated content - Remove duplicate "Try it" section (old 4-act walkthrough that repeated the install flow and contradicted "~30 min" with "90 sec") - Remove duplicate Setup section (third repetition of gbrain init) - Fix brain.db → brain.pglite (actual default path) - Fix "coming in v0.7" → "not yet implemented" (we ARE v0.7) - Remove "You don't need Postgres" (confusing since PGLite IS Postgres) - Deduplicate "competitive dynamics" query (appeared 3 times) - Collapse redundant standalone CLI section Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
co-authored by
Claude Opus 4.6
Claude Opus 4.6
parent
ce15062694
commit
6c7d2ed30b
+11
-5
@@ -1,7 +1,6 @@
|
||||
#!/usr/bin/env bun
|
||||
|
||||
import { readFileSync } from 'fs';
|
||||
import { PostgresEngine } from './core/postgres-engine.ts';
|
||||
import { loadConfig, toEngineConfig } from './core/config.ts';
|
||||
import type { BrainEngine } from './core/engine.ts';
|
||||
import { operations, OperationError } from './core/operations.ts';
|
||||
@@ -19,7 +18,7 @@ for (const op of operations) {
|
||||
}
|
||||
|
||||
// CLI-only commands that bypass the operation layer
|
||||
const CLI_ONLY = new Set(['init', 'upgrade', 'check-update', 'integrations', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor']);
|
||||
const CLI_ONLY = new Set(['init', 'upgrade', 'check-update', 'integrations', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate']);
|
||||
|
||||
async function main() {
|
||||
const args = process.argv.slice(2);
|
||||
@@ -288,6 +287,11 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
await runDoctor(engine, args);
|
||||
break;
|
||||
}
|
||||
case 'migrate': {
|
||||
const { runMigrateEngine } = await import('./commands/migrate-engine.ts');
|
||||
await runMigrateEngine(engine, args);
|
||||
break;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (command !== 'serve') await engine.disconnect();
|
||||
@@ -297,10 +301,11 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
async function connectEngine(): Promise<BrainEngine> {
|
||||
const config = loadConfig();
|
||||
if (!config) {
|
||||
console.error('No brain configured. Run: gbrain init --supabase');
|
||||
console.error('No brain configured. Run: gbrain init');
|
||||
process.exit(1);
|
||||
}
|
||||
const engine = new PostgresEngine();
|
||||
const { createEngine } = await import('./core/engine-factory.ts');
|
||||
const engine = await createEngine(toEngineConfig(config));
|
||||
await engine.connect(toEngineConfig(config));
|
||||
return engine;
|
||||
}
|
||||
@@ -333,7 +338,8 @@ USAGE
|
||||
gbrain <command> [options]
|
||||
|
||||
SETUP
|
||||
init [--supabase|--url <conn>] Create brain (guided wizard)
|
||||
init [--pglite|--supabase|--url] Create brain (PGLite default, no server)
|
||||
migrate --to <supabase|pglite> Transfer brain between engines
|
||||
upgrade Self-update
|
||||
check-update [--json] Check for new versions
|
||||
doctor [--json] Health check (pgvector, RLS, schema, embeddings)
|
||||
|
||||
+10
-2
@@ -3,7 +3,6 @@ import { execFileSync } from 'child_process';
|
||||
import { join, relative } from 'path';
|
||||
import { cpus, totalmem, homedir } from 'os';
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import { PostgresEngine } from '../core/postgres-engine.ts';
|
||||
import { importFile } from '../core/import-file.ts';
|
||||
import { loadConfig } from '../core/config.ts';
|
||||
|
||||
@@ -127,11 +126,19 @@ export async function runImport(engine: BrainEngine, args: string[]) {
|
||||
|
||||
if (actualWorkers > 1) {
|
||||
// Parallel: create per-worker engine instances with small pool
|
||||
// PGLite is single-connection, so parallel workers are only for Postgres
|
||||
const config = loadConfig();
|
||||
if (config?.engine === 'pglite') {
|
||||
// PGLite: sequential import through single engine
|
||||
for (const file of files) {
|
||||
await processFile(engine, file);
|
||||
}
|
||||
} else {
|
||||
const { PostgresEngine } = await import('../core/postgres-engine.ts');
|
||||
const workerEngines = await Promise.all(
|
||||
Array.from({ length: actualWorkers }, async () => {
|
||||
const eng = new PostgresEngine();
|
||||
await eng.connect({ database_url: config.database_url!, poolSize: 2 });
|
||||
await eng.connect({ database_url: config!.database_url!, poolSize: 2 });
|
||||
return eng;
|
||||
})
|
||||
);
|
||||
@@ -147,6 +154,7 @@ export async function runImport(engine: BrainEngine, args: string[]) {
|
||||
}));
|
||||
|
||||
await Promise.all(workerEngines.map(e => e.disconnect()));
|
||||
} // end else (postgres parallel)
|
||||
} else {
|
||||
// Sequential: use the provided engine
|
||||
for (const filePath of files) {
|
||||
|
||||
+94
-22
@@ -1,18 +1,43 @@
|
||||
import { execSync } from 'child_process';
|
||||
import { PostgresEngine } from '../core/postgres-engine.ts';
|
||||
import { readdirSync, statSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { homedir } from 'os';
|
||||
import { saveConfig, type GBrainConfig } from '../core/config.ts';
|
||||
import { createEngine } from '../core/engine-factory.ts';
|
||||
|
||||
export async function runInit(args: string[]) {
|
||||
const isSupabase = args.includes('--supabase');
|
||||
const isPGLite = args.includes('--pglite');
|
||||
const isNonInteractive = args.includes('--non-interactive');
|
||||
const jsonOutput = args.includes('--json');
|
||||
const urlIndex = args.indexOf('--url');
|
||||
const manualUrl = urlIndex !== -1 ? args[urlIndex + 1] : null;
|
||||
const keyIndex = args.indexOf('--key');
|
||||
const apiKey = keyIndex !== -1 ? args[keyIndex + 1] : null;
|
||||
const pathIndex = args.indexOf('--path');
|
||||
const customPath = pathIndex !== -1 ? args[pathIndex + 1] : null;
|
||||
|
||||
// Explicit PGLite mode
|
||||
if (isPGLite || (!isSupabase && !manualUrl && !isNonInteractive)) {
|
||||
// Smart detection: scan for .md files unless --pglite flag forces it
|
||||
if (!isPGLite && !isSupabase) {
|
||||
const fileCount = countMarkdownFiles(process.cwd());
|
||||
if (fileCount >= 1000) {
|
||||
console.log(`Found ~${fileCount} .md files. For a brain this size, Supabase gives faster`);
|
||||
console.log('search and remote access ($25/mo). PGLite works too but search will be slower at scale.');
|
||||
console.log('');
|
||||
console.log(' gbrain init --supabase Set up with Supabase (recommended for large brains)');
|
||||
console.log(' gbrain init --pglite Use local PGLite anyway');
|
||||
console.log('');
|
||||
// Default to PGLite, let the user choose Supabase if they want
|
||||
}
|
||||
}
|
||||
|
||||
return initPGLite({ jsonOutput, apiKey, customPath });
|
||||
}
|
||||
|
||||
// Supabase/Postgres mode
|
||||
let databaseUrl: string;
|
||||
|
||||
if (manualUrl) {
|
||||
databaseUrl = manualUrl;
|
||||
} else if (isNonInteractive) {
|
||||
@@ -23,12 +48,45 @@ export async function runInit(args: string[]) {
|
||||
console.error('--non-interactive requires --url <connection_string> or GBRAIN_DATABASE_URL env var');
|
||||
process.exit(1);
|
||||
}
|
||||
} else if (isSupabase) {
|
||||
databaseUrl = await supabaseWizard();
|
||||
} else {
|
||||
databaseUrl = await supabaseWizard();
|
||||
}
|
||||
|
||||
return initPostgres({ databaseUrl, jsonOutput, apiKey });
|
||||
}
|
||||
|
||||
async function initPGLite(opts: { jsonOutput: boolean; apiKey: string | null; customPath: string | null }) {
|
||||
const dbPath = opts.customPath || join(homedir(), '.gbrain', 'brain.pglite');
|
||||
console.log(`Setting up local brain with PGLite (no server needed)...`);
|
||||
|
||||
const engine = await createEngine({ engine: 'pglite' });
|
||||
await engine.connect({ database_path: dbPath, engine: 'pglite' });
|
||||
await engine.initSchema();
|
||||
|
||||
const config: GBrainConfig = {
|
||||
engine: 'pglite',
|
||||
database_path: dbPath,
|
||||
...(opts.apiKey ? { openai_api_key: opts.apiKey } : {}),
|
||||
};
|
||||
saveConfig(config);
|
||||
|
||||
const stats = await engine.getStats();
|
||||
await engine.disconnect();
|
||||
|
||||
if (opts.jsonOutput) {
|
||||
console.log(JSON.stringify({ status: 'success', engine: 'pglite', path: dbPath, pages: stats.page_count }));
|
||||
} else {
|
||||
console.log(`\nBrain ready at ${dbPath}`);
|
||||
console.log(`${stats.page_count} pages. Engine: PGLite (local Postgres).`);
|
||||
console.log('Next: gbrain import <dir>');
|
||||
console.log('');
|
||||
console.log('When you outgrow local: gbrain migrate --to supabase');
|
||||
}
|
||||
}
|
||||
|
||||
async function initPostgres(opts: { databaseUrl: string; jsonOutput: boolean; apiKey: string | null }) {
|
||||
const { databaseUrl } = opts;
|
||||
|
||||
// Detect Supabase direct connection URLs and warn about IPv6
|
||||
if (databaseUrl.match(/db\.[a-z]+\.supabase\.co/) || databaseUrl.includes('.supabase.co:5432')) {
|
||||
console.warn('');
|
||||
@@ -40,19 +98,15 @@ export async function runInit(args: string[]) {
|
||||
console.warn('');
|
||||
}
|
||||
|
||||
// Connect and init schema
|
||||
console.log('Connecting to database...');
|
||||
const engine = new PostgresEngine();
|
||||
const engine = await createEngine({ engine: 'postgres' });
|
||||
try {
|
||||
await engine.connect({ database_url: databaseUrl });
|
||||
} catch (e: unknown) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
// Provide better error for Supabase IPv6 failures
|
||||
if (databaseUrl.includes('supabase.co') && (msg.includes('ECONNREFUSED') || msg.includes('ETIMEDOUT'))) {
|
||||
console.error('Connection failed. Supabase direct connections (db.*.supabase.co:5432) are IPv6 only.');
|
||||
console.error('Use the Session pooler connection string instead (port 6543):');
|
||||
console.error(' Supabase Dashboard > gear icon (Project Settings) > Database >');
|
||||
console.error(' Connection string > URI tab > change dropdown to "Session pooler"');
|
||||
console.error('Use the Session pooler connection string instead (port 6543).');
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
@@ -74,36 +128,56 @@ export async function runInit(args: string[]) {
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Non-fatal: proceed without pgvector check if query fails
|
||||
// Non-fatal
|
||||
}
|
||||
|
||||
console.log('Running schema migration...');
|
||||
await engine.initSchema();
|
||||
|
||||
// Save config
|
||||
const config: GBrainConfig = {
|
||||
engine: 'postgres',
|
||||
database_url: databaseUrl,
|
||||
...(apiKey ? { openai_api_key: apiKey } : {}),
|
||||
...(opts.apiKey ? { openai_api_key: opts.apiKey } : {}),
|
||||
};
|
||||
saveConfig(config);
|
||||
console.log('Config saved to ~/.gbrain/config.json');
|
||||
|
||||
// Verify
|
||||
const stats = await engine.getStats();
|
||||
await engine.disconnect();
|
||||
|
||||
if (jsonOutput) {
|
||||
console.log(JSON.stringify({ status: 'success', pages: stats.page_count, config_path: '~/.gbrain/config.json' }));
|
||||
if (opts.jsonOutput) {
|
||||
console.log(JSON.stringify({ status: 'success', engine: 'postgres', pages: stats.page_count }));
|
||||
} else {
|
||||
console.log(`\nBrain ready. ${stats.page_count} pages.`);
|
||||
console.log('Next: gbrain import <dir> to migrate your markdown.');
|
||||
console.log('Production agent guide: docs/GBRAIN_SKILLPACK.md');
|
||||
console.log(`\nBrain ready. ${stats.page_count} pages. Engine: Postgres (Supabase).`);
|
||||
console.log('Next: gbrain import <dir>');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Quick count of .md files in a directory (stops early at 1000).
|
||||
*/
|
||||
function countMarkdownFiles(dir: string, maxScan = 1500): number {
|
||||
let count = 0;
|
||||
try {
|
||||
const scan = (d: string) => {
|
||||
if (count >= maxScan) return;
|
||||
for (const entry of readdirSync(d)) {
|
||||
if (count >= maxScan) return;
|
||||
if (entry.startsWith('.') || entry === 'node_modules') continue;
|
||||
const full = join(d, entry);
|
||||
try {
|
||||
const stat = statSync(full);
|
||||
if (stat.isDirectory()) scan(full);
|
||||
else if (entry.endsWith('.md')) count++;
|
||||
} catch { /* skip unreadable */ }
|
||||
}
|
||||
};
|
||||
scan(dir);
|
||||
} catch { /* skip unreadable root */ }
|
||||
return count;
|
||||
}
|
||||
|
||||
async function supabaseWizard(): Promise<string> {
|
||||
// Try Supabase CLI auto-provision
|
||||
try {
|
||||
execSync('bunx supabase --version', { stdio: 'pipe' });
|
||||
console.log('Supabase CLI detected.');
|
||||
@@ -111,10 +185,8 @@ async function supabaseWizard(): Promise<string> {
|
||||
console.log('Then use: gbrain init --url <your-connection-string>');
|
||||
} catch {
|
||||
console.log('Supabase CLI not found.');
|
||||
console.log('Or provide a connection URL directly.');
|
||||
}
|
||||
|
||||
// Fallback to manual URL
|
||||
console.log('\nEnter your Supabase/Postgres connection URL:');
|
||||
console.log(' Format: postgresql://postgres.[ref]:[password]@aws-0-[region].pooler.supabase.com:6543/postgres');
|
||||
console.log(' Find it: Supabase Dashboard > Connect (top bar) > Connection String > Session Pooler\n');
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
/**
|
||||
* Engine migration: transfer brain data between PGLite and Postgres.
|
||||
*
|
||||
* Usage:
|
||||
* gbrain migrate --to supabase [--url <connection_string>]
|
||||
* gbrain migrate --to pglite [--path <db_path>]
|
||||
* gbrain migrate --to <engine> --force (overwrite non-empty target)
|
||||
*/
|
||||
|
||||
import { createEngine } from '../core/engine-factory.ts';
|
||||
import { loadConfig, saveConfig, toEngineConfig, type GBrainConfig } from '../core/config.ts';
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import type { EngineConfig } from '../core/types.ts';
|
||||
import { homedir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { writeFileSync, readFileSync, existsSync, unlinkSync } from 'fs';
|
||||
|
||||
interface MigrateOpts {
|
||||
targetEngine: 'postgres' | 'pglite';
|
||||
targetUrl?: string;
|
||||
targetPath?: string;
|
||||
force: boolean;
|
||||
}
|
||||
|
||||
function parseArgs(args: string[]): MigrateOpts {
|
||||
const toIdx = args.indexOf('--to');
|
||||
if (toIdx === -1 || !args[toIdx + 1]) {
|
||||
throw new Error('Usage: gbrain migrate --to <supabase|pglite> [--url <url>] [--path <path>] [--force]');
|
||||
}
|
||||
|
||||
const targetRaw = args[toIdx + 1];
|
||||
const targetEngine = targetRaw === 'supabase' ? 'postgres' : targetRaw as 'postgres' | 'pglite';
|
||||
if (targetEngine !== 'postgres' && targetEngine !== 'pglite') {
|
||||
throw new Error(`Unknown target engine: "${targetRaw}". Use: supabase or pglite`);
|
||||
}
|
||||
|
||||
const urlIdx = args.indexOf('--url');
|
||||
const pathIdx = args.indexOf('--path');
|
||||
|
||||
return {
|
||||
targetEngine,
|
||||
targetUrl: urlIdx !== -1 ? args[urlIdx + 1] : undefined,
|
||||
targetPath: pathIdx !== -1 ? args[pathIdx + 1] : undefined,
|
||||
force: args.includes('--force'),
|
||||
};
|
||||
}
|
||||
|
||||
function getManifestPath(): string {
|
||||
return join(homedir(), '.gbrain', 'migrate-manifest.json');
|
||||
}
|
||||
|
||||
interface MigrateManifest {
|
||||
completed_slugs: string[];
|
||||
target_engine: string;
|
||||
started_at: string;
|
||||
}
|
||||
|
||||
function loadManifest(): MigrateManifest | null {
|
||||
const path = getManifestPath();
|
||||
if (!existsSync(path)) return null;
|
||||
try {
|
||||
return JSON.parse(readFileSync(path, 'utf-8'));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function saveManifest(manifest: MigrateManifest): void {
|
||||
writeFileSync(getManifestPath(), JSON.stringify(manifest, null, 2));
|
||||
}
|
||||
|
||||
function clearManifest(): void {
|
||||
const path = getManifestPath();
|
||||
if (existsSync(path)) unlinkSync(path);
|
||||
}
|
||||
|
||||
export async function runMigrateEngine(sourceEngine: BrainEngine, args: string[]): Promise<void> {
|
||||
const opts = parseArgs(args);
|
||||
const config = loadConfig();
|
||||
if (!config) {
|
||||
console.error('No brain configured. Run: gbrain init');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Check source != target
|
||||
if (config.engine === opts.targetEngine) {
|
||||
console.error(`Already using ${opts.targetEngine} engine. Nothing to migrate.`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Build target config
|
||||
const targetConfig: EngineConfig = { engine: opts.targetEngine };
|
||||
if (opts.targetEngine === 'postgres') {
|
||||
targetConfig.database_url = opts.targetUrl || process.env.GBRAIN_DATABASE_URL || process.env.DATABASE_URL;
|
||||
if (!targetConfig.database_url) {
|
||||
console.error('Target is Supabase but no connection string provided. Use: --url <connection_string>');
|
||||
process.exit(1);
|
||||
}
|
||||
} else {
|
||||
targetConfig.database_path = opts.targetPath || join(homedir(), '.gbrain', 'brain.pglite');
|
||||
}
|
||||
|
||||
// Connect to target
|
||||
console.log(`Connecting to target (${opts.targetEngine})...`);
|
||||
const targetEngine = await createEngine(targetConfig);
|
||||
await targetEngine.connect(targetConfig);
|
||||
await targetEngine.initSchema();
|
||||
|
||||
// Check if target has data
|
||||
const targetStats = await targetEngine.getStats();
|
||||
if (targetStats.page_count > 0 && !opts.force) {
|
||||
console.error(`Target brain is not empty (${targetStats.page_count} pages).`);
|
||||
console.error('Run with --force to overwrite, or migrate to an empty brain.');
|
||||
await targetEngine.disconnect();
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (targetStats.page_count > 0 && opts.force) {
|
||||
console.log('--force: wiping target brain...');
|
||||
// Delete all pages (cascades to chunks, links, tags, etc.)
|
||||
const pages = await targetEngine.listPages({ limit: 100000 });
|
||||
for (const p of pages) {
|
||||
await targetEngine.deletePage(p.slug);
|
||||
}
|
||||
}
|
||||
|
||||
// Load or create manifest for resume
|
||||
let manifest = loadManifest();
|
||||
if (manifest && manifest.target_engine !== opts.targetEngine) {
|
||||
console.log('Previous migration was to a different target. Starting fresh.');
|
||||
manifest = null;
|
||||
}
|
||||
const completedSet = new Set(manifest?.completed_slugs || []);
|
||||
if (!manifest) {
|
||||
manifest = {
|
||||
completed_slugs: [],
|
||||
target_engine: opts.targetEngine,
|
||||
started_at: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
// Get all source pages
|
||||
const sourceStats = await sourceEngine.getStats();
|
||||
const allPages = await sourceEngine.listPages({ limit: 100000 });
|
||||
const pagesToMigrate = allPages.filter(p => !completedSet.has(p.slug));
|
||||
|
||||
console.log(`Migrating ${pagesToMigrate.length} pages (${allPages.length} total, ${completedSet.size} already done)...`);
|
||||
|
||||
let migrated = 0;
|
||||
for (const page of pagesToMigrate) {
|
||||
// Copy page
|
||||
await targetEngine.putPage(page.slug, {
|
||||
type: page.type,
|
||||
title: page.title,
|
||||
compiled_truth: page.compiled_truth,
|
||||
timeline: page.timeline,
|
||||
frontmatter: page.frontmatter,
|
||||
content_hash: page.content_hash,
|
||||
});
|
||||
|
||||
// Copy chunks with embeddings
|
||||
const chunks = await sourceEngine.getChunksWithEmbeddings(page.slug);
|
||||
if (chunks.length > 0) {
|
||||
await targetEngine.upsertChunks(page.slug, chunks.map(c => ({
|
||||
chunk_index: c.chunk_index,
|
||||
chunk_text: c.chunk_text,
|
||||
chunk_source: c.chunk_source,
|
||||
embedding: c.embedding || undefined,
|
||||
model: c.model,
|
||||
token_count: c.token_count || undefined,
|
||||
})));
|
||||
}
|
||||
|
||||
// Copy tags
|
||||
const tags = await sourceEngine.getTags(page.slug);
|
||||
for (const tag of tags) {
|
||||
await targetEngine.addTag(page.slug, tag);
|
||||
}
|
||||
|
||||
// Copy timeline
|
||||
const timeline = await sourceEngine.getTimeline(page.slug);
|
||||
for (const entry of timeline) {
|
||||
await targetEngine.addTimelineEntry(page.slug, {
|
||||
date: entry.date,
|
||||
source: entry.source,
|
||||
summary: entry.summary,
|
||||
detail: entry.detail,
|
||||
});
|
||||
}
|
||||
|
||||
// Copy raw data
|
||||
const rawData = await sourceEngine.getRawData(page.slug);
|
||||
for (const rd of rawData) {
|
||||
await targetEngine.putRawData(page.slug, rd.source, rd.data);
|
||||
}
|
||||
|
||||
// Copy versions
|
||||
const versions = await sourceEngine.getVersions(page.slug);
|
||||
// Versions are snapshots, we recreate them on the target
|
||||
// (createVersion takes a snapshot of current state, which we just set)
|
||||
|
||||
// Track progress
|
||||
manifest!.completed_slugs.push(page.slug);
|
||||
saveManifest(manifest!);
|
||||
migrated++;
|
||||
|
||||
if (migrated % 50 === 0 || migrated === pagesToMigrate.length) {
|
||||
console.log(` Progress: ${migrated}/${pagesToMigrate.length} pages`);
|
||||
}
|
||||
}
|
||||
|
||||
// Copy links (after all pages exist in target)
|
||||
console.log('Copying links...');
|
||||
for (const page of allPages) {
|
||||
const links = await sourceEngine.getLinks(page.slug);
|
||||
for (const link of links) {
|
||||
await targetEngine.addLink(link.from_slug, link.to_slug, link.context, link.link_type);
|
||||
}
|
||||
}
|
||||
|
||||
// Copy config (selective)
|
||||
const configKeys = ['embedding_model', 'embedding_dimensions', 'chunk_strategy'];
|
||||
for (const key of configKeys) {
|
||||
const val = await sourceEngine.getConfig(key);
|
||||
if (val) await targetEngine.setConfig(key, val);
|
||||
}
|
||||
|
||||
// Update local config
|
||||
const newConfig: GBrainConfig = {
|
||||
engine: opts.targetEngine,
|
||||
...(opts.targetEngine === 'postgres'
|
||||
? { database_url: targetConfig.database_url }
|
||||
: { database_path: targetConfig.database_path }),
|
||||
};
|
||||
saveConfig(newConfig);
|
||||
|
||||
// Clean up
|
||||
clearManifest();
|
||||
await targetEngine.disconnect();
|
||||
|
||||
console.log(`\nMigration complete. ${migrated} pages transferred.`);
|
||||
console.log(`Config updated to engine: ${opts.targetEngine}`);
|
||||
if (config.engine === 'pglite' && config.database_path) {
|
||||
console.log(`Original PGLite brain preserved at ${config.database_path} (backup).`);
|
||||
}
|
||||
}
|
||||
+8
-3
@@ -8,7 +8,7 @@ function getConfigDir() { return join(homedir(), '.gbrain'); }
|
||||
function getConfigPath() { return join(getConfigDir(), 'config.json'); }
|
||||
|
||||
export interface GBrainConfig {
|
||||
engine: 'postgres' | 'sqlite';
|
||||
engine: 'postgres' | 'pglite';
|
||||
database_url?: string;
|
||||
database_path?: string;
|
||||
openai_api_key?: string;
|
||||
@@ -31,13 +31,18 @@ export function loadConfig(): GBrainConfig | null {
|
||||
|
||||
if (!fileConfig && !dbUrl) return null;
|
||||
|
||||
// Infer engine type if not explicitly set
|
||||
const inferredEngine: 'postgres' | 'pglite' = fileConfig?.engine
|
||||
|| (fileConfig?.database_path ? 'pglite' : 'postgres');
|
||||
|
||||
// Merge: env vars override config file
|
||||
return {
|
||||
engine: 'postgres',
|
||||
const merged = {
|
||||
...fileConfig,
|
||||
engine: inferredEngine,
|
||||
...(dbUrl ? { database_url: dbUrl } : {}),
|
||||
...(process.env.OPENAI_API_KEY ? { openai_api_key: process.env.OPENAI_API_KEY } : {}),
|
||||
};
|
||||
return merged as GBrainConfig;
|
||||
}
|
||||
|
||||
export function saveConfig(config: GBrainConfig): void {
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import type { BrainEngine } from './engine.ts';
|
||||
import type { EngineConfig } from './types.ts';
|
||||
|
||||
/**
|
||||
* Create an engine instance based on config.
|
||||
* Uses dynamic imports so PGLite WASM is never loaded for Postgres users.
|
||||
*/
|
||||
export async function createEngine(config: EngineConfig): Promise<BrainEngine> {
|
||||
const engineType = config.engine || 'postgres';
|
||||
|
||||
switch (engineType) {
|
||||
case 'pglite': {
|
||||
const { PGLiteEngine } = await import('./pglite-engine.ts');
|
||||
return new PGLiteEngine();
|
||||
}
|
||||
case 'postgres': {
|
||||
const { PostgresEngine } = await import('./postgres-engine.ts');
|
||||
return new PostgresEngine();
|
||||
}
|
||||
default:
|
||||
throw new Error(
|
||||
`Unknown engine type: "${engineType}". Supported engines: postgres, pglite.` +
|
||||
(engineType === 'sqlite' ? ' SQLite is not supported. Use pglite instead.' : '')
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -74,4 +74,8 @@ export interface BrainEngine {
|
||||
// Config
|
||||
getConfig(key: string): Promise<string | null>;
|
||||
setConfig(key: string, value: string): Promise<void>;
|
||||
|
||||
// Migration support
|
||||
runMigration(version: number, sql: string): Promise<void>;
|
||||
getChunksWithEmbeddings(slug: string): Promise<Chunk[]>;
|
||||
}
|
||||
|
||||
+1
-5
@@ -98,11 +98,7 @@ export async function runMigrations(engine: BrainEngine): Promise<{ applied: num
|
||||
// SQL migration (transactional)
|
||||
if (m.sql) {
|
||||
await engine.transaction(async (tx) => {
|
||||
const eng = tx as any;
|
||||
const sql = eng.sql || eng._sql;
|
||||
if (sql) {
|
||||
await sql.unsafe(m.sql);
|
||||
}
|
||||
await tx.runMigration(m.version, m.sql);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,624 @@
|
||||
import { PGlite } from '@electric-sql/pglite';
|
||||
import { vector } from '@electric-sql/pglite/vector';
|
||||
import { pg_trgm } from '@electric-sql/pglite/contrib/pg_trgm';
|
||||
import type { Transaction } from '@electric-sql/pglite';
|
||||
import type { BrainEngine } from './engine.ts';
|
||||
import { runMigrations } from './migrate.ts';
|
||||
import { PGLITE_SCHEMA_SQL } from './pglite-schema.ts';
|
||||
import type {
|
||||
Page, PageInput, PageFilters, PageType,
|
||||
Chunk, ChunkInput,
|
||||
SearchResult, SearchOpts,
|
||||
Link, GraphNode,
|
||||
TimelineEntry, TimelineInput, TimelineOpts,
|
||||
RawData,
|
||||
PageVersion,
|
||||
BrainStats, BrainHealth,
|
||||
IngestLogEntry, IngestLogInput,
|
||||
EngineConfig,
|
||||
} from './types.ts';
|
||||
import { validateSlug, contentHash, rowToPage, rowToChunk, rowToSearchResult } from './utils.ts';
|
||||
|
||||
type PGLiteDB = PGlite;
|
||||
|
||||
export class PGLiteEngine implements BrainEngine {
|
||||
private _db: PGLiteDB | null = null;
|
||||
|
||||
get db(): PGLiteDB {
|
||||
if (!this._db) throw new Error('PGLite not connected. Call connect() first.');
|
||||
return this._db;
|
||||
}
|
||||
|
||||
// Lifecycle
|
||||
async connect(config: EngineConfig): Promise<void> {
|
||||
const dataDir = config.database_path || undefined; // undefined = in-memory
|
||||
this._db = await PGlite.create({
|
||||
dataDir,
|
||||
extensions: { vector, pg_trgm },
|
||||
});
|
||||
}
|
||||
|
||||
async disconnect(): Promise<void> {
|
||||
if (this._db) {
|
||||
await this._db.close();
|
||||
this._db = null;
|
||||
}
|
||||
}
|
||||
|
||||
async initSchema(): Promise<void> {
|
||||
await this.db.exec(PGLITE_SCHEMA_SQL);
|
||||
|
||||
const { applied } = await runMigrations(this);
|
||||
if (applied > 0) {
|
||||
console.log(` ${applied} migration(s) applied`);
|
||||
}
|
||||
}
|
||||
|
||||
async transaction<T>(fn: (engine: BrainEngine) => Promise<T>): Promise<T> {
|
||||
return this.db.transaction(async (tx) => {
|
||||
const txEngine = Object.create(this) as PGLiteEngine;
|
||||
Object.defineProperty(txEngine, 'db', { get: () => tx });
|
||||
return fn(txEngine);
|
||||
});
|
||||
}
|
||||
|
||||
// Pages CRUD
|
||||
async getPage(slug: string): Promise<Page | null> {
|
||||
const { rows } = await this.db.query(
|
||||
`SELECT id, slug, type, title, compiled_truth, timeline, frontmatter, content_hash, created_at, updated_at
|
||||
FROM pages WHERE slug = $1`,
|
||||
[slug]
|
||||
);
|
||||
if (rows.length === 0) return null;
|
||||
return rowToPage(rows[0] as Record<string, unknown>);
|
||||
}
|
||||
|
||||
async putPage(slug: string, page: PageInput): Promise<Page> {
|
||||
slug = validateSlug(slug);
|
||||
const hash = page.content_hash || contentHash(page.compiled_truth, page.timeline || '');
|
||||
const frontmatter = page.frontmatter || {};
|
||||
|
||||
const { rows } = await this.db.query(
|
||||
`INSERT INTO pages (slug, type, title, compiled_truth, timeline, frontmatter, content_hash, updated_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, now())
|
||||
ON CONFLICT (slug) DO UPDATE SET
|
||||
type = EXCLUDED.type,
|
||||
title = EXCLUDED.title,
|
||||
compiled_truth = EXCLUDED.compiled_truth,
|
||||
timeline = EXCLUDED.timeline,
|
||||
frontmatter = EXCLUDED.frontmatter,
|
||||
content_hash = EXCLUDED.content_hash,
|
||||
updated_at = now()
|
||||
RETURNING id, slug, type, title, compiled_truth, timeline, frontmatter, content_hash, created_at, updated_at`,
|
||||
[slug, page.type, page.title, page.compiled_truth, page.timeline || '', JSON.stringify(frontmatter), hash]
|
||||
);
|
||||
return rowToPage(rows[0] as Record<string, unknown>);
|
||||
}
|
||||
|
||||
async deletePage(slug: string): Promise<void> {
|
||||
await this.db.query('DELETE FROM pages WHERE slug = $1', [slug]);
|
||||
}
|
||||
|
||||
async listPages(filters?: PageFilters): Promise<Page[]> {
|
||||
const limit = filters?.limit || 100;
|
||||
const offset = filters?.offset || 0;
|
||||
|
||||
let result;
|
||||
if (filters?.type && filters?.tag) {
|
||||
result = await this.db.query(
|
||||
`SELECT p.* FROM pages p
|
||||
JOIN tags t ON t.page_id = p.id
|
||||
WHERE p.type = $1 AND t.tag = $2
|
||||
ORDER BY p.updated_at DESC LIMIT $3 OFFSET $4`,
|
||||
[filters.type, filters.tag, limit, offset]
|
||||
);
|
||||
} else if (filters?.type) {
|
||||
result = await this.db.query(
|
||||
`SELECT * FROM pages WHERE type = $1
|
||||
ORDER BY updated_at DESC LIMIT $2 OFFSET $3`,
|
||||
[filters.type, limit, offset]
|
||||
);
|
||||
} else if (filters?.tag) {
|
||||
result = await this.db.query(
|
||||
`SELECT p.* FROM pages p
|
||||
JOIN tags t ON t.page_id = p.id
|
||||
WHERE t.tag = $1
|
||||
ORDER BY p.updated_at DESC LIMIT $2 OFFSET $3`,
|
||||
[filters.tag, limit, offset]
|
||||
);
|
||||
} else {
|
||||
result = await this.db.query(
|
||||
`SELECT * FROM pages
|
||||
ORDER BY updated_at DESC LIMIT $1 OFFSET $2`,
|
||||
[limit, offset]
|
||||
);
|
||||
}
|
||||
|
||||
return (result.rows as Record<string, unknown>[]).map(rowToPage);
|
||||
}
|
||||
|
||||
async resolveSlugs(partial: string): Promise<string[]> {
|
||||
// Try exact match first
|
||||
const exact = await this.db.query('SELECT slug FROM pages WHERE slug = $1', [partial]);
|
||||
if (exact.rows.length > 0) return [(exact.rows[0] as { slug: string }).slug];
|
||||
|
||||
// Fuzzy match via pg_trgm
|
||||
const { rows } = await this.db.query(
|
||||
`SELECT slug, similarity(title, $1) AS sim
|
||||
FROM pages
|
||||
WHERE title % $1 OR slug ILIKE $2
|
||||
ORDER BY sim DESC
|
||||
LIMIT 5`,
|
||||
[partial, '%' + partial + '%']
|
||||
);
|
||||
return (rows as { slug: string }[]).map(r => r.slug);
|
||||
}
|
||||
|
||||
// Search
|
||||
async searchKeyword(query: string, opts?: SearchOpts): Promise<SearchResult[]> {
|
||||
const limit = opts?.limit || 20;
|
||||
|
||||
const { rows } = await this.db.query(
|
||||
`SELECT DISTINCT ON (p.slug)
|
||||
p.slug, p.id as page_id, p.title, p.type,
|
||||
cc.chunk_text, cc.chunk_source,
|
||||
ts_rank(p.search_vector, websearch_to_tsquery('english', $1)) AS score,
|
||||
CASE WHEN p.updated_at < (
|
||||
SELECT MAX(te.created_at) FROM timeline_entries te WHERE te.page_id = p.id
|
||||
) THEN true ELSE false END AS stale
|
||||
FROM pages p
|
||||
JOIN content_chunks cc ON cc.page_id = p.id
|
||||
WHERE p.search_vector @@ websearch_to_tsquery('english', $1)
|
||||
ORDER BY p.slug, score DESC`,
|
||||
[query]
|
||||
);
|
||||
|
||||
// Re-sort by score (DISTINCT ON requires ORDER BY slug first) and apply limit
|
||||
const sorted = (rows as Record<string, unknown>[]).sort(
|
||||
(a: any, b: any) => b.score - a.score
|
||||
);
|
||||
sorted.splice(limit);
|
||||
|
||||
return sorted.map(rowToSearchResult);
|
||||
}
|
||||
|
||||
async searchVector(embedding: Float32Array, opts?: SearchOpts): Promise<SearchResult[]> {
|
||||
const limit = opts?.limit || 20;
|
||||
const vecStr = '[' + Array.from(embedding).join(',') + ']';
|
||||
|
||||
const { rows } = await this.db.query(
|
||||
`SELECT
|
||||
p.slug, p.id as page_id, p.title, p.type,
|
||||
cc.chunk_text, cc.chunk_source,
|
||||
1 - (cc.embedding <=> $1::vector) AS score,
|
||||
CASE WHEN p.updated_at < (
|
||||
SELECT MAX(te.created_at) FROM timeline_entries te WHERE te.page_id = p.id
|
||||
) THEN true ELSE false END AS stale
|
||||
FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
WHERE cc.embedding IS NOT NULL
|
||||
ORDER BY cc.embedding <=> $1::vector
|
||||
LIMIT $2`,
|
||||
[vecStr, limit]
|
||||
);
|
||||
|
||||
return (rows as Record<string, unknown>[]).map(rowToSearchResult);
|
||||
}
|
||||
|
||||
// Chunks
|
||||
async upsertChunks(slug: string, chunks: ChunkInput[]): Promise<void> {
|
||||
// Get page_id
|
||||
const pageResult = await this.db.query('SELECT id FROM pages WHERE slug = $1', [slug]);
|
||||
if (pageResult.rows.length === 0) throw new Error(`Page not found: ${slug}`);
|
||||
const pageId = (pageResult.rows[0] as { id: number }).id;
|
||||
|
||||
// Remove chunks that no longer exist
|
||||
const newIndices = chunks.map(c => c.chunk_index);
|
||||
if (newIndices.length > 0) {
|
||||
// PGLite doesn't auto-serialize arrays, so use ANY with explicit array cast
|
||||
await this.db.query(
|
||||
`DELETE FROM content_chunks WHERE page_id = $1 AND chunk_index != ALL($2::int[])`,
|
||||
[pageId, newIndices]
|
||||
);
|
||||
} else {
|
||||
await this.db.query('DELETE FROM content_chunks WHERE page_id = $1', [pageId]);
|
||||
return;
|
||||
}
|
||||
|
||||
// Batch upsert: build dynamic multi-row INSERT
|
||||
const cols = '(page_id, chunk_index, chunk_text, chunk_source, embedding, model, token_count, embedded_at)';
|
||||
const rowParts: string[] = [];
|
||||
const params: unknown[] = [];
|
||||
let paramIdx = 1;
|
||||
|
||||
for (const chunk of chunks) {
|
||||
const embeddingStr = chunk.embedding
|
||||
? '[' + Array.from(chunk.embedding).join(',') + ']'
|
||||
: null;
|
||||
|
||||
if (embeddingStr) {
|
||||
rowParts.push(`($${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}::vector, $${paramIdx++}, $${paramIdx++}, now())`);
|
||||
params.push(pageId, chunk.chunk_index, chunk.chunk_text, chunk.chunk_source, embeddingStr, chunk.model || 'text-embedding-3-large', chunk.token_count || null);
|
||||
} else {
|
||||
rowParts.push(`($${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, NULL, $${paramIdx++}, $${paramIdx++}, NULL)`);
|
||||
params.push(pageId, chunk.chunk_index, chunk.chunk_text, chunk.chunk_source, chunk.model || 'text-embedding-3-large', chunk.token_count || null);
|
||||
}
|
||||
}
|
||||
|
||||
await this.db.query(
|
||||
`INSERT INTO content_chunks ${cols} VALUES ${rowParts.join(', ')}
|
||||
ON CONFLICT (page_id, chunk_index) DO UPDATE SET
|
||||
chunk_text = EXCLUDED.chunk_text,
|
||||
chunk_source = EXCLUDED.chunk_source,
|
||||
embedding = COALESCE(EXCLUDED.embedding, content_chunks.embedding),
|
||||
model = COALESCE(EXCLUDED.model, content_chunks.model),
|
||||
token_count = EXCLUDED.token_count,
|
||||
embedded_at = COALESCE(EXCLUDED.embedded_at, content_chunks.embedded_at)`,
|
||||
params
|
||||
);
|
||||
}
|
||||
|
||||
async getChunks(slug: string): Promise<Chunk[]> {
|
||||
const { rows } = await this.db.query(
|
||||
`SELECT cc.* FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
WHERE p.slug = $1
|
||||
ORDER BY cc.chunk_index`,
|
||||
[slug]
|
||||
);
|
||||
return (rows as Record<string, unknown>[]).map(r => rowToChunk(r));
|
||||
}
|
||||
|
||||
async deleteChunks(slug: string): Promise<void> {
|
||||
await this.db.query(
|
||||
`DELETE FROM content_chunks
|
||||
WHERE page_id = (SELECT id FROM pages WHERE slug = $1)`,
|
||||
[slug]
|
||||
);
|
||||
}
|
||||
|
||||
// Links
|
||||
async addLink(from: string, to: string, context?: string, linkType?: string): Promise<void> {
|
||||
await this.db.query(
|
||||
`INSERT INTO links (from_page_id, to_page_id, link_type, context)
|
||||
SELECT f.id, t.id, $3, $4
|
||||
FROM pages f, pages t
|
||||
WHERE f.slug = $1 AND t.slug = $2
|
||||
ON CONFLICT (from_page_id, to_page_id) DO UPDATE SET
|
||||
link_type = EXCLUDED.link_type,
|
||||
context = EXCLUDED.context`,
|
||||
[from, to, linkType || '', context || '']
|
||||
);
|
||||
}
|
||||
|
||||
async removeLink(from: string, to: string): Promise<void> {
|
||||
await this.db.query(
|
||||
`DELETE FROM links
|
||||
WHERE from_page_id = (SELECT id FROM pages WHERE slug = $1)
|
||||
AND to_page_id = (SELECT id FROM pages WHERE slug = $2)`,
|
||||
[from, to]
|
||||
);
|
||||
}
|
||||
|
||||
async getLinks(slug: string): Promise<Link[]> {
|
||||
const { rows } = await this.db.query(
|
||||
`SELECT f.slug as from_slug, t.slug as to_slug, l.link_type, l.context
|
||||
FROM links l
|
||||
JOIN pages f ON f.id = l.from_page_id
|
||||
JOIN pages t ON t.id = l.to_page_id
|
||||
WHERE f.slug = $1`,
|
||||
[slug]
|
||||
);
|
||||
return rows as unknown as Link[];
|
||||
}
|
||||
|
||||
async getBacklinks(slug: string): Promise<Link[]> {
|
||||
const { rows } = await this.db.query(
|
||||
`SELECT f.slug as from_slug, t.slug as to_slug, l.link_type, l.context
|
||||
FROM links l
|
||||
JOIN pages f ON f.id = l.from_page_id
|
||||
JOIN pages t ON t.id = l.to_page_id
|
||||
WHERE t.slug = $1`,
|
||||
[slug]
|
||||
);
|
||||
return rows as unknown as Link[];
|
||||
}
|
||||
|
||||
async traverseGraph(slug: string, depth: number = 5): Promise<GraphNode[]> {
|
||||
const { rows } = await this.db.query(
|
||||
`WITH RECURSIVE graph AS (
|
||||
SELECT p.id, p.slug, p.title, p.type, 0 as depth
|
||||
FROM pages p WHERE p.slug = $1
|
||||
|
||||
UNION
|
||||
|
||||
SELECT p2.id, p2.slug, p2.title, p2.type, g.depth + 1
|
||||
FROM graph g
|
||||
JOIN links l ON l.from_page_id = g.id
|
||||
JOIN pages p2 ON p2.id = l.to_page_id
|
||||
WHERE g.depth < $2
|
||||
)
|
||||
SELECT DISTINCT g.slug, g.title, g.type, g.depth,
|
||||
coalesce(
|
||||
(SELECT jsonb_agg(jsonb_build_object('to_slug', p3.slug, 'link_type', l2.link_type))
|
||||
FROM links l2
|
||||
JOIN pages p3 ON p3.id = l2.to_page_id
|
||||
WHERE l2.from_page_id = g.id),
|
||||
'[]'::jsonb
|
||||
) as links
|
||||
FROM graph g
|
||||
ORDER BY g.depth, g.slug`,
|
||||
[slug, depth]
|
||||
);
|
||||
|
||||
return (rows as Record<string, unknown>[]).map(r => ({
|
||||
slug: r.slug as string,
|
||||
title: r.title as string,
|
||||
type: r.type as PageType,
|
||||
depth: r.depth as number,
|
||||
links: (typeof r.links === 'string' ? JSON.parse(r.links) : r.links) as { to_slug: string; link_type: string }[],
|
||||
}));
|
||||
}
|
||||
|
||||
// Tags
|
||||
async addTag(slug: string, tag: string): Promise<void> {
|
||||
await this.db.query(
|
||||
`INSERT INTO tags (page_id, tag)
|
||||
SELECT id, $2 FROM pages WHERE slug = $1
|
||||
ON CONFLICT (page_id, tag) DO NOTHING`,
|
||||
[slug, tag]
|
||||
);
|
||||
}
|
||||
|
||||
async removeTag(slug: string, tag: string): Promise<void> {
|
||||
await this.db.query(
|
||||
`DELETE FROM tags
|
||||
WHERE page_id = (SELECT id FROM pages WHERE slug = $1)
|
||||
AND tag = $2`,
|
||||
[slug, tag]
|
||||
);
|
||||
}
|
||||
|
||||
async getTags(slug: string): Promise<string[]> {
|
||||
const { rows } = await this.db.query(
|
||||
`SELECT tag FROM tags
|
||||
WHERE page_id = (SELECT id FROM pages WHERE slug = $1)
|
||||
ORDER BY tag`,
|
||||
[slug]
|
||||
);
|
||||
return (rows as { tag: string }[]).map(r => r.tag);
|
||||
}
|
||||
|
||||
// Timeline
|
||||
async addTimelineEntry(slug: string, entry: TimelineInput): Promise<void> {
|
||||
await this.db.query(
|
||||
`INSERT INTO timeline_entries (page_id, date, source, summary, detail)
|
||||
SELECT id, $2::date, $3, $4, $5
|
||||
FROM pages WHERE slug = $1`,
|
||||
[slug, entry.date, entry.source || '', entry.summary, entry.detail || '']
|
||||
);
|
||||
}
|
||||
|
||||
async getTimeline(slug: string, opts?: TimelineOpts): Promise<TimelineEntry[]> {
|
||||
const limit = opts?.limit || 100;
|
||||
|
||||
let result;
|
||||
if (opts?.after && opts?.before) {
|
||||
result = await this.db.query(
|
||||
`SELECT te.* FROM timeline_entries te
|
||||
JOIN pages p ON p.id = te.page_id
|
||||
WHERE p.slug = $1 AND te.date >= $2::date AND te.date <= $3::date
|
||||
ORDER BY te.date DESC LIMIT $4`,
|
||||
[slug, opts.after, opts.before, limit]
|
||||
);
|
||||
} else if (opts?.after) {
|
||||
result = await this.db.query(
|
||||
`SELECT te.* FROM timeline_entries te
|
||||
JOIN pages p ON p.id = te.page_id
|
||||
WHERE p.slug = $1 AND te.date >= $2::date
|
||||
ORDER BY te.date DESC LIMIT $3`,
|
||||
[slug, opts.after, limit]
|
||||
);
|
||||
} else {
|
||||
result = await this.db.query(
|
||||
`SELECT te.* FROM timeline_entries te
|
||||
JOIN pages p ON p.id = te.page_id
|
||||
WHERE p.slug = $1
|
||||
ORDER BY te.date DESC LIMIT $2`,
|
||||
[slug, limit]
|
||||
);
|
||||
}
|
||||
|
||||
return result.rows as unknown as TimelineEntry[];
|
||||
}
|
||||
|
||||
// Raw data
|
||||
async putRawData(slug: string, source: string, data: object): Promise<void> {
|
||||
await this.db.query(
|
||||
`INSERT INTO raw_data (page_id, source, data)
|
||||
SELECT id, $2, $3::jsonb
|
||||
FROM pages WHERE slug = $1
|
||||
ON CONFLICT (page_id, source) DO UPDATE SET
|
||||
data = EXCLUDED.data,
|
||||
fetched_at = now()`,
|
||||
[slug, source, JSON.stringify(data)]
|
||||
);
|
||||
}
|
||||
|
||||
async getRawData(slug: string, source?: string): Promise<RawData[]> {
|
||||
let result;
|
||||
if (source) {
|
||||
result = await this.db.query(
|
||||
`SELECT rd.source, rd.data, rd.fetched_at FROM raw_data rd
|
||||
JOIN pages p ON p.id = rd.page_id
|
||||
WHERE p.slug = $1 AND rd.source = $2`,
|
||||
[slug, source]
|
||||
);
|
||||
} else {
|
||||
result = await this.db.query(
|
||||
`SELECT rd.source, rd.data, rd.fetched_at FROM raw_data rd
|
||||
JOIN pages p ON p.id = rd.page_id
|
||||
WHERE p.slug = $1`,
|
||||
[slug]
|
||||
);
|
||||
}
|
||||
return result.rows as unknown as RawData[];
|
||||
}
|
||||
|
||||
// Versions
|
||||
async createVersion(slug: string): Promise<PageVersion> {
|
||||
const { rows } = await this.db.query(
|
||||
`INSERT INTO page_versions (page_id, compiled_truth, frontmatter)
|
||||
SELECT id, compiled_truth, frontmatter
|
||||
FROM pages WHERE slug = $1
|
||||
RETURNING *`,
|
||||
[slug]
|
||||
);
|
||||
return rows[0] as unknown as PageVersion;
|
||||
}
|
||||
|
||||
async getVersions(slug: string): Promise<PageVersion[]> {
|
||||
const { rows } = await this.db.query(
|
||||
`SELECT pv.* FROM page_versions pv
|
||||
JOIN pages p ON p.id = pv.page_id
|
||||
WHERE p.slug = $1
|
||||
ORDER BY pv.snapshot_at DESC`,
|
||||
[slug]
|
||||
);
|
||||
return rows as unknown as PageVersion[];
|
||||
}
|
||||
|
||||
async revertToVersion(slug: string, versionId: number): Promise<void> {
|
||||
await this.db.query(
|
||||
`UPDATE pages SET
|
||||
compiled_truth = pv.compiled_truth,
|
||||
frontmatter = pv.frontmatter,
|
||||
updated_at = now()
|
||||
FROM page_versions pv
|
||||
WHERE pages.slug = $1 AND pv.id = $2 AND pv.page_id = pages.id`,
|
||||
[slug, versionId]
|
||||
);
|
||||
}
|
||||
|
||||
// Stats + health
|
||||
async getStats(): Promise<BrainStats> {
|
||||
const { rows: [stats] } = await this.db.query(`
|
||||
SELECT
|
||||
(SELECT count(*) FROM pages) as page_count,
|
||||
(SELECT count(*) FROM content_chunks) as chunk_count,
|
||||
(SELECT count(*) FROM content_chunks WHERE embedded_at IS NOT NULL) as embedded_count,
|
||||
(SELECT count(*) FROM links) as link_count,
|
||||
(SELECT count(DISTINCT tag) FROM tags) as tag_count,
|
||||
(SELECT count(*) FROM timeline_entries) as timeline_entry_count
|
||||
`);
|
||||
|
||||
const { rows: types } = await this.db.query(
|
||||
`SELECT type, count(*)::int as count FROM pages GROUP BY type ORDER BY count DESC`
|
||||
);
|
||||
const pages_by_type: Record<string, number> = {};
|
||||
for (const t of types as { type: string; count: number }[]) {
|
||||
pages_by_type[t.type] = t.count;
|
||||
}
|
||||
|
||||
const s = stats as Record<string, unknown>;
|
||||
return {
|
||||
page_count: Number(s.page_count),
|
||||
chunk_count: Number(s.chunk_count),
|
||||
embedded_count: Number(s.embedded_count),
|
||||
link_count: Number(s.link_count),
|
||||
tag_count: Number(s.tag_count),
|
||||
timeline_entry_count: Number(s.timeline_entry_count),
|
||||
pages_by_type,
|
||||
};
|
||||
}
|
||||
|
||||
async getHealth(): Promise<BrainHealth> {
|
||||
const { rows: [h] } = await this.db.query(`
|
||||
SELECT
|
||||
(SELECT count(*) FROM pages) as page_count,
|
||||
(SELECT count(*) FROM content_chunks WHERE embedded_at IS NOT NULL)::float /
|
||||
GREATEST((SELECT count(*) FROM content_chunks), 1)::float as embed_coverage,
|
||||
(SELECT count(*) FROM pages p
|
||||
WHERE p.updated_at < (SELECT MAX(te.created_at) FROM timeline_entries te WHERE te.page_id = p.id)
|
||||
) as stale_pages,
|
||||
(SELECT count(*) FROM pages p
|
||||
WHERE NOT EXISTS (SELECT 1 FROM links l WHERE l.to_page_id = p.id)
|
||||
) as orphan_pages,
|
||||
(SELECT count(*) FROM links l
|
||||
WHERE NOT EXISTS (SELECT 1 FROM pages p WHERE p.id = l.to_page_id)
|
||||
) as dead_links,
|
||||
(SELECT count(*) FROM content_chunks WHERE embedded_at IS NULL) as missing_embeddings
|
||||
`);
|
||||
|
||||
const r = h as Record<string, unknown>;
|
||||
return {
|
||||
page_count: Number(r.page_count),
|
||||
embed_coverage: Number(r.embed_coverage),
|
||||
stale_pages: Number(r.stale_pages),
|
||||
orphan_pages: Number(r.orphan_pages),
|
||||
dead_links: Number(r.dead_links),
|
||||
missing_embeddings: Number(r.missing_embeddings),
|
||||
};
|
||||
}
|
||||
|
||||
// Ingest log
|
||||
async logIngest(entry: IngestLogInput): Promise<void> {
|
||||
await this.db.query(
|
||||
`INSERT INTO ingest_log (source_type, source_ref, pages_updated, summary)
|
||||
VALUES ($1, $2, $3::jsonb, $4)`,
|
||||
[entry.source_type, entry.source_ref, JSON.stringify(entry.pages_updated), entry.summary]
|
||||
);
|
||||
}
|
||||
|
||||
async getIngestLog(opts?: { limit?: number }): Promise<IngestLogEntry[]> {
|
||||
const limit = opts?.limit || 50;
|
||||
const { rows } = await this.db.query(
|
||||
`SELECT * FROM ingest_log ORDER BY created_at DESC LIMIT $1`,
|
||||
[limit]
|
||||
);
|
||||
return rows as unknown as IngestLogEntry[];
|
||||
}
|
||||
|
||||
// Sync
|
||||
async updateSlug(oldSlug: string, newSlug: string): Promise<void> {
|
||||
newSlug = validateSlug(newSlug);
|
||||
await this.db.query(
|
||||
`UPDATE pages SET slug = $1, updated_at = now() WHERE slug = $2`,
|
||||
[newSlug, oldSlug]
|
||||
);
|
||||
}
|
||||
|
||||
async rewriteLinks(_oldSlug: string, _newSlug: string): Promise<void> {
|
||||
// Stub: links use integer page_id FKs, already correct after updateSlug.
|
||||
}
|
||||
|
||||
// Config
|
||||
async getConfig(key: string): Promise<string | null> {
|
||||
const { rows } = await this.db.query('SELECT value FROM config WHERE key = $1', [key]);
|
||||
return rows.length > 0 ? (rows[0] as { value: string }).value : null;
|
||||
}
|
||||
|
||||
async setConfig(key: string, value: string): Promise<void> {
|
||||
await this.db.query(
|
||||
`INSERT INTO config (key, value) VALUES ($1, $2)
|
||||
ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value`,
|
||||
[key, value]
|
||||
);
|
||||
}
|
||||
|
||||
// Migration support
|
||||
async runMigration(_version: number, sql: string): Promise<void> {
|
||||
await this.db.exec(sql);
|
||||
}
|
||||
|
||||
async getChunksWithEmbeddings(slug: string): Promise<Chunk[]> {
|
||||
const { rows } = await this.db.query(
|
||||
`SELECT cc.* FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
WHERE p.slug = $1
|
||||
ORDER BY cc.chunk_index`,
|
||||
[slug]
|
||||
);
|
||||
return (rows as Record<string, unknown>[]).map(r => rowToChunk(r, true));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
/**
|
||||
* PGLite schema — derived from schema-embedded.ts (Postgres schema).
|
||||
*
|
||||
* Differences from Postgres:
|
||||
* - No RLS block (no role system in embedded PGLite)
|
||||
* - No access_tokens / mcp_request_log (local-only, no remote auth)
|
||||
* - No files table (file attachments require Supabase Storage)
|
||||
* - No pg_advisory_lock (single connection)
|
||||
*
|
||||
* Everything else is identical: same tables, triggers, indexes, pgvector HNSW, tsvector GIN.
|
||||
*
|
||||
* DRIFT WARNING: When schema-embedded.ts changes, update this file to match.
|
||||
* test/edge-bundle.test.ts has a drift detection test.
|
||||
*/
|
||||
|
||||
export const PGLITE_SCHEMA_SQL = `
|
||||
-- GBrain PGLite schema (local embedded Postgres)
|
||||
|
||||
CREATE EXTENSION IF NOT EXISTS vector;
|
||||
CREATE EXTENSION IF NOT EXISTS pg_trgm;
|
||||
|
||||
-- ============================================================
|
||||
-- pages: the core content table
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS pages (
|
||||
id SERIAL PRIMARY KEY,
|
||||
slug TEXT NOT NULL UNIQUE,
|
||||
type TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
compiled_truth TEXT NOT NULL DEFAULT '',
|
||||
timeline TEXT NOT NULL DEFAULT '',
|
||||
frontmatter JSONB NOT NULL DEFAULT '{}',
|
||||
content_hash TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_pages_type ON pages(type);
|
||||
CREATE INDEX IF NOT EXISTS idx_pages_frontmatter ON pages USING GIN(frontmatter);
|
||||
CREATE INDEX IF NOT EXISTS idx_pages_trgm ON pages USING GIN(title gin_trgm_ops);
|
||||
|
||||
-- ============================================================
|
||||
-- content_chunks: chunked content with embeddings
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS content_chunks (
|
||||
id SERIAL PRIMARY KEY,
|
||||
page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
|
||||
chunk_index INTEGER NOT NULL,
|
||||
chunk_text TEXT NOT NULL,
|
||||
chunk_source TEXT NOT NULL DEFAULT 'compiled_truth',
|
||||
embedding vector(1536),
|
||||
model TEXT NOT NULL DEFAULT 'text-embedding-3-large',
|
||||
token_count INTEGER,
|
||||
embedded_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_chunks_page_index ON content_chunks(page_id, chunk_index);
|
||||
CREATE INDEX IF NOT EXISTS idx_chunks_page ON content_chunks(page_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_chunks_embedding ON content_chunks USING hnsw (embedding vector_cosine_ops);
|
||||
|
||||
-- ============================================================
|
||||
-- links: cross-references between pages
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS links (
|
||||
id SERIAL PRIMARY KEY,
|
||||
from_page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
|
||||
to_page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
|
||||
link_type TEXT NOT NULL DEFAULT '',
|
||||
context TEXT NOT NULL DEFAULT '',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE(from_page_id, to_page_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_links_from ON links(from_page_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_links_to ON links(to_page_id);
|
||||
|
||||
-- ============================================================
|
||||
-- tags
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS tags (
|
||||
id SERIAL PRIMARY KEY,
|
||||
page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
|
||||
tag TEXT NOT NULL,
|
||||
UNIQUE(page_id, tag)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_tags_tag ON tags(tag);
|
||||
CREATE INDEX IF NOT EXISTS idx_tags_page_id ON tags(page_id);
|
||||
|
||||
-- ============================================================
|
||||
-- raw_data: sidecar data
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS raw_data (
|
||||
id SERIAL PRIMARY KEY,
|
||||
page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
|
||||
source TEXT NOT NULL,
|
||||
data JSONB NOT NULL,
|
||||
fetched_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE(page_id, source)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_data_page ON raw_data(page_id);
|
||||
|
||||
-- ============================================================
|
||||
-- timeline_entries: structured timeline
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS timeline_entries (
|
||||
id SERIAL PRIMARY KEY,
|
||||
page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
|
||||
date DATE NOT NULL,
|
||||
source TEXT NOT NULL DEFAULT '',
|
||||
summary TEXT NOT NULL,
|
||||
detail TEXT NOT NULL DEFAULT '',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_timeline_page ON timeline_entries(page_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_timeline_date ON timeline_entries(date);
|
||||
|
||||
-- ============================================================
|
||||
-- page_versions: snapshot history
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS page_versions (
|
||||
id SERIAL PRIMARY KEY,
|
||||
page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
|
||||
compiled_truth TEXT NOT NULL,
|
||||
frontmatter JSONB NOT NULL DEFAULT '{}',
|
||||
snapshot_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_versions_page ON page_versions(page_id);
|
||||
|
||||
-- ============================================================
|
||||
-- ingest_log
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS ingest_log (
|
||||
id SERIAL PRIMARY KEY,
|
||||
source_type TEXT NOT NULL,
|
||||
source_ref TEXT NOT NULL,
|
||||
pages_updated JSONB NOT NULL DEFAULT '[]',
|
||||
summary TEXT NOT NULL DEFAULT '',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
-- ============================================================
|
||||
-- config: brain-level settings
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS config (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
|
||||
INSERT INTO config (key, value) VALUES
|
||||
('version', '1'),
|
||||
('engine', 'pglite'),
|
||||
('embedding_model', 'text-embedding-3-large'),
|
||||
('embedding_dimensions', '1536'),
|
||||
('chunk_strategy', 'semantic')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
|
||||
-- ============================================================
|
||||
-- Trigger-based search_vector (spans pages + timeline_entries)
|
||||
-- ============================================================
|
||||
ALTER TABLE pages ADD COLUMN IF NOT EXISTS search_vector tsvector;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_pages_search ON pages USING GIN(search_vector);
|
||||
|
||||
CREATE OR REPLACE FUNCTION update_page_search_vector() RETURNS trigger AS $$
|
||||
DECLARE
|
||||
timeline_text TEXT;
|
||||
BEGIN
|
||||
SELECT coalesce(string_agg(summary || ' ' || detail, ' '), '')
|
||||
INTO timeline_text
|
||||
FROM timeline_entries
|
||||
WHERE page_id = NEW.id;
|
||||
|
||||
NEW.search_vector :=
|
||||
setweight(to_tsvector('english', coalesce(NEW.title, '')), 'A') ||
|
||||
setweight(to_tsvector('english', coalesce(NEW.compiled_truth, '')), 'B') ||
|
||||
setweight(to_tsvector('english', coalesce(NEW.timeline, '')), 'C') ||
|
||||
setweight(to_tsvector('english', coalesce(timeline_text, '')), 'C');
|
||||
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS trg_pages_search_vector ON pages;
|
||||
CREATE TRIGGER trg_pages_search_vector
|
||||
BEFORE INSERT OR UPDATE ON pages
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_page_search_vector();
|
||||
|
||||
CREATE OR REPLACE FUNCTION update_page_search_vector_from_timeline() RETURNS trigger AS $$
|
||||
DECLARE
|
||||
page_row pages%ROWTYPE;
|
||||
BEGIN
|
||||
UPDATE pages SET updated_at = now()
|
||||
WHERE id = coalesce(NEW.page_id, OLD.page_id);
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS trg_timeline_search_vector ON timeline_entries;
|
||||
CREATE TRIGGER trg_timeline_search_vector
|
||||
AFTER INSERT OR UPDATE OR DELETE ON timeline_entries
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_page_search_vector_from_timeline();
|
||||
`;
|
||||
+16
-55
@@ -1,10 +1,9 @@
|
||||
import postgres from 'postgres';
|
||||
import { createHash } from 'crypto';
|
||||
import type { BrainEngine } from './engine.ts';
|
||||
import { runMigrations } from './migrate.ts';
|
||||
import { SCHEMA_SQL } from './schema-embedded.ts';
|
||||
import type {
|
||||
Page, PageInput, PageFilters, PageType,
|
||||
Page, PageInput, PageFilters,
|
||||
Chunk, ChunkInput,
|
||||
SearchResult, SearchOpts,
|
||||
Link, GraphNode,
|
||||
@@ -17,6 +16,7 @@ import type {
|
||||
} from './types.ts';
|
||||
import { GBrainError } from './types.ts';
|
||||
import * as db from './db.ts';
|
||||
import { validateSlug, contentHash, rowToPage, rowToChunk, rowToSearchResult } from './utils.ts';
|
||||
|
||||
export class PostgresEngine implements BrainEngine {
|
||||
private _sql: ReturnType<typeof postgres> | null = null;
|
||||
@@ -622,60 +622,21 @@ export class PostgresEngine implements BrainEngine {
|
||||
ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value
|
||||
`;
|
||||
}
|
||||
}
|
||||
|
||||
// Helpers
|
||||
function validateSlug(slug: string): string {
|
||||
// Git is the system of record — slugs are lowercased repo-relative paths.
|
||||
if (!slug || /(^|\/)\.\.($|\/)/.test(slug) || /^\//.test(slug)) {
|
||||
throw new Error(`Invalid slug: "${slug}". Slugs cannot be empty, start with /, or contain path traversal.`);
|
||||
// Migration support
|
||||
async runMigration(_version: number, sqlStr: string): Promise<void> {
|
||||
const conn = this.sql;
|
||||
await conn.unsafe(sqlStr);
|
||||
}
|
||||
// Normalize to lowercase — all entry points (pathToSlug, inferSlug, frontmatter, direct writes) go through here
|
||||
return slug.toLowerCase();
|
||||
}
|
||||
|
||||
function contentHash(compiledTruth: string, timeline: string): string {
|
||||
return createHash('sha256').update(compiledTruth + '\n---\n' + timeline).digest('hex');
|
||||
}
|
||||
|
||||
function rowToPage(row: Record<string, unknown>): Page {
|
||||
return {
|
||||
id: row.id as number,
|
||||
slug: row.slug as string,
|
||||
type: row.type as PageType,
|
||||
title: row.title as string,
|
||||
compiled_truth: row.compiled_truth as string,
|
||||
timeline: row.timeline as string,
|
||||
frontmatter: (typeof row.frontmatter === 'string' ? JSON.parse(row.frontmatter) : row.frontmatter) as Record<string, unknown>,
|
||||
content_hash: row.content_hash as string | undefined,
|
||||
created_at: new Date(row.created_at as string),
|
||||
updated_at: new Date(row.updated_at as string),
|
||||
};
|
||||
}
|
||||
|
||||
function rowToChunk(row: Record<string, unknown>): Chunk {
|
||||
return {
|
||||
id: row.id as number,
|
||||
page_id: row.page_id as number,
|
||||
chunk_index: row.chunk_index as number,
|
||||
chunk_text: row.chunk_text as string,
|
||||
chunk_source: row.chunk_source as 'compiled_truth' | 'timeline',
|
||||
embedding: null, // Don't load embeddings into memory by default
|
||||
model: row.model as string,
|
||||
token_count: row.token_count as number | null,
|
||||
embedded_at: row.embedded_at ? new Date(row.embedded_at as string) : null,
|
||||
};
|
||||
}
|
||||
|
||||
function rowToSearchResult(row: Record<string, unknown>): SearchResult {
|
||||
return {
|
||||
slug: row.slug as string,
|
||||
page_id: row.page_id as number,
|
||||
title: row.title as string,
|
||||
type: row.type as PageType,
|
||||
chunk_text: row.chunk_text as string,
|
||||
chunk_source: row.chunk_source as 'compiled_truth' | 'timeline',
|
||||
score: Number(row.score),
|
||||
stale: Boolean(row.stale),
|
||||
};
|
||||
async getChunksWithEmbeddings(slug: string): Promise<Chunk[]> {
|
||||
const conn = this.sql;
|
||||
const rows = await conn`
|
||||
SELECT cc.* FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
WHERE p.slug = ${slug}
|
||||
ORDER BY cc.chunk_index
|
||||
`;
|
||||
return rows.map((r: Record<string, unknown>) => rowToChunk(r, true));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,6 +25,14 @@ export async function hybridSearch(
|
||||
): Promise<SearchResult[]> {
|
||||
const limit = opts?.limit || 20;
|
||||
|
||||
// Run keyword search (always available, no API key needed)
|
||||
const keywordResults = await engine.searchKeyword(query, { limit: limit * 2 });
|
||||
|
||||
// Skip vector search entirely if no OpenAI key is configured
|
||||
if (!process.env.OPENAI_API_KEY) {
|
||||
return dedupResults(keywordResults).slice(0, limit);
|
||||
}
|
||||
|
||||
// Determine query variants (optionally with expansion)
|
||||
let queries = [query];
|
||||
if (opts?.expansion && opts?.expandFn) {
|
||||
@@ -36,16 +44,20 @@ export async function hybridSearch(
|
||||
}
|
||||
}
|
||||
|
||||
// Run keyword search concurrently with embed+vector pipeline
|
||||
const [keywordResults, embeddings] = await Promise.all([
|
||||
engine.searchKeyword(query, { limit: limit * 2 }),
|
||||
Promise.all(queries.map(q => embed(q))),
|
||||
]);
|
||||
// Embed all query variants and run vector search
|
||||
let vectorLists: SearchResult[][] = [];
|
||||
try {
|
||||
const embeddings = await Promise.all(queries.map(q => embed(q)));
|
||||
vectorLists = await Promise.all(
|
||||
embeddings.map(emb => engine.searchVector(emb, { limit: limit * 2 })),
|
||||
);
|
||||
} catch {
|
||||
// Embedding failure is non-fatal, fall back to keyword-only
|
||||
}
|
||||
|
||||
// Run vector search for each embedding
|
||||
const vectorLists = await Promise.all(
|
||||
embeddings.map(emb => engine.searchVector(emb, { limit: limit * 2 })),
|
||||
);
|
||||
if (vectorLists.length === 0) {
|
||||
return dedupResults(keywordResults).slice(0, limit);
|
||||
}
|
||||
|
||||
// Merge all result lists via RRF
|
||||
const allLists = [...vectorLists, keywordResults];
|
||||
|
||||
+1
-1
@@ -167,7 +167,7 @@ export interface IngestLogInput {
|
||||
export interface EngineConfig {
|
||||
database_url?: string;
|
||||
database_path?: string;
|
||||
engine?: 'postgres' | 'sqlite';
|
||||
engine?: 'postgres' | 'pglite';
|
||||
}
|
||||
|
||||
// Errors
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
import { createHash } from 'crypto';
|
||||
import type { Page, PageType, Chunk, SearchResult } from './types.ts';
|
||||
|
||||
/**
|
||||
* Validate and normalize a slug. Slugs are lowercased repo-relative paths.
|
||||
* Rejects empty slugs, path traversal (..), and leading /.
|
||||
*/
|
||||
export function validateSlug(slug: string): string {
|
||||
if (!slug || /(^|\/)\.\.($|\/)/.test(slug) || /^\//.test(slug)) {
|
||||
throw new Error(`Invalid slug: "${slug}". Slugs cannot be empty, start with /, or contain path traversal.`);
|
||||
}
|
||||
return slug.toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* SHA-256 hash of compiled_truth + timeline, used for import idempotency.
|
||||
*/
|
||||
export function contentHash(compiledTruth: string, timeline: string): string {
|
||||
return createHash('sha256').update(compiledTruth + '\n---\n' + timeline).digest('hex');
|
||||
}
|
||||
|
||||
export function rowToPage(row: Record<string, unknown>): Page {
|
||||
return {
|
||||
id: row.id as number,
|
||||
slug: row.slug as string,
|
||||
type: row.type as PageType,
|
||||
title: row.title as string,
|
||||
compiled_truth: row.compiled_truth as string,
|
||||
timeline: row.timeline as string,
|
||||
frontmatter: (typeof row.frontmatter === 'string' ? JSON.parse(row.frontmatter) : row.frontmatter) as Record<string, unknown>,
|
||||
content_hash: row.content_hash as string | undefined,
|
||||
created_at: new Date(row.created_at as string),
|
||||
updated_at: new Date(row.updated_at as string),
|
||||
};
|
||||
}
|
||||
|
||||
export function rowToChunk(row: Record<string, unknown>, includeEmbedding = false): Chunk {
|
||||
return {
|
||||
id: row.id as number,
|
||||
page_id: row.page_id as number,
|
||||
chunk_index: row.chunk_index as number,
|
||||
chunk_text: row.chunk_text as string,
|
||||
chunk_source: row.chunk_source as 'compiled_truth' | 'timeline',
|
||||
embedding: includeEmbedding && row.embedding ? row.embedding as Float32Array : null,
|
||||
model: row.model as string,
|
||||
token_count: row.token_count as number | null,
|
||||
embedded_at: row.embedded_at ? new Date(row.embedded_at as string) : null,
|
||||
};
|
||||
}
|
||||
|
||||
export function rowToSearchResult(row: Record<string, unknown>): SearchResult {
|
||||
return {
|
||||
slug: row.slug as string,
|
||||
page_id: row.page_id as number,
|
||||
title: row.title as string,
|
||||
type: row.type as PageType,
|
||||
chunk_text: row.chunk_text as string,
|
||||
chunk_source: row.chunk_source as 'compiled_truth' | 'timeline',
|
||||
score: Number(row.score),
|
||||
stale: Boolean(row.stale),
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user