diff --git a/package.json b/package.json index d15b0c60b..1e8dc0431 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "dev": "bun run src/cli.ts", "build": "bun build --compile --outfile bin/gbrain src/cli.ts", "build:all": "bun build --compile --target=bun-darwin-arm64 --outfile bin/gbrain-darwin-arm64 src/cli.ts && bun build --compile --target=bun-linux-x64 --outfile bin/gbrain-linux-x64 src/cli.ts", + "build:schema": "bash scripts/build-schema.sh", "test": "bun test", "test:e2e": "bun test test/e2e/", "prepublish:clawhub": "bun run build:all", diff --git a/scripts/build-schema.sh b/scripts/build-schema.sh new file mode 100755 index 000000000..e6f66f7d4 --- /dev/null +++ b/scripts/build-schema.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Generate src/core/schema-embedded.ts from src/schema.sql +# One source of truth: schema.sql is the canonical file. +# This script produces a TypeScript constant for use in compiled binaries and Edge Functions. +set -e +SCHEMA_FILE="src/schema.sql" +OUT_FILE="src/core/schema-embedded.ts" +echo "// AUTO-GENERATED — do not edit. Run: bun run build:schema" > "$OUT_FILE" +echo "// Source: $SCHEMA_FILE" >> "$OUT_FILE" +echo "" >> "$OUT_FILE" +echo "export const SCHEMA_SQL = \`" >> "$OUT_FILE" +# Escape backticks and dollar signs in the SQL for template literal safety +sed 's/`/\\`/g; s/\$/\\$/g' "$SCHEMA_FILE" >> "$OUT_FILE" +echo "\`;" >> "$OUT_FILE" +echo "Generated $OUT_FILE from $SCHEMA_FILE" diff --git a/src/core/db.ts b/src/core/db.ts index 459a4834d..36b4a6cd4 100644 --- a/src/core/db.ts +++ b/src/core/db.ts @@ -1,7 +1,6 @@ import postgres from 'postgres'; -import { readFileSync } from 'fs'; -import { join, dirname } from 'path'; import { GBrainError, type EngineConfig } from './types.ts'; +import { SCHEMA_SQL } from './schema-embedded.ts'; let sql: ReturnType | null = null; @@ -61,15 +60,7 @@ export async function disconnect(): Promise { export async function initSchema(): Promise { const conn = getConnection(); - - // Read schema SQL and execute as a single statement. - // The postgres driver handles multi-statement SQL natively, including - // PL/pgSQL functions with $$ delimiter blocks that contain semicolons. - // The schema uses IF NOT EXISTS / CREATE OR REPLACE for idempotency. - const schemaPath = join(dirname(new URL(import.meta.url).pathname), '..', 'schema.sql'); - const schemaSql = readFileSync(schemaPath, 'utf-8'); - - await conn.unsafe(schemaSql); + await conn.unsafe(SCHEMA_SQL); } export async function withTransaction(fn: (tx: ReturnType) => Promise): Promise { diff --git a/src/core/postgres-engine.ts b/src/core/postgres-engine.ts index 3428fe52d..f2fc3c071 100644 --- a/src/core/postgres-engine.ts +++ b/src/core/postgres-engine.ts @@ -1,9 +1,8 @@ import postgres from 'postgres'; import { createHash } from 'crypto'; -import { readFileSync } from 'fs'; -import { join, dirname } from 'path'; import type { BrainEngine } from './engine.ts'; import { runMigrations } from './migrate.ts'; +import { SCHEMA_SQL } from './schema-embedded.ts'; import type { Page, PageInput, PageFilters, PageType, Chunk, ChunkInput, @@ -58,9 +57,7 @@ export class PostgresEngine implements BrainEngine { async initSchema(): Promise { const conn = this.sql; - const schemaPath = join(dirname(new URL(import.meta.url).pathname), '..', 'schema.sql'); - const schemaSql = readFileSync(schemaPath, 'utf-8'); - await conn.unsafe(schemaSql); + await conn.unsafe(SCHEMA_SQL); // Run any pending migrations automatically const { applied } = await runMigrations(this); diff --git a/src/core/schema-embedded.ts b/src/core/schema-embedded.ts new file mode 100644 index 000000000..7ec5ae69b --- /dev/null +++ b/src/core/schema-embedded.ts @@ -0,0 +1,252 @@ +// AUTO-GENERATED — do not edit. Run: bun run build:schema +// Source: src/schema.sql + +export const SCHEMA_SQL = ` +-- GBrain Postgres + pgvector schema + +CREATE EXTENSION IF NOT EXISTS vector; +CREATE EXTENSION IF NOT EXISTS pg_trgm; + +-- ============================================================ +-- pages: the core content table +-- ============================================================ +CREATE TABLE IF NOT EXISTS pages ( + id SERIAL PRIMARY KEY, + slug TEXT NOT NULL UNIQUE, + type TEXT NOT NULL, + title TEXT NOT NULL, + compiled_truth TEXT NOT NULL DEFAULT '', + timeline TEXT NOT NULL DEFAULT '', + frontmatter JSONB NOT NULL DEFAULT '{}', + content_hash TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_pages_type ON pages(type); +CREATE INDEX IF NOT EXISTS idx_pages_frontmatter ON pages USING GIN(frontmatter); +CREATE INDEX IF NOT EXISTS idx_pages_trgm ON pages USING GIN(title gin_trgm_ops); + +-- ============================================================ +-- content_chunks: chunked content with embeddings +-- ============================================================ +CREATE TABLE IF NOT EXISTS content_chunks ( + id SERIAL PRIMARY KEY, + page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE, + chunk_index INTEGER NOT NULL, + chunk_text TEXT NOT NULL, + chunk_source TEXT NOT NULL DEFAULT 'compiled_truth', + embedding vector(1536), + model TEXT NOT NULL DEFAULT 'text-embedding-3-large', + token_count INTEGER, + embedded_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_chunks_page_index ON content_chunks(page_id, chunk_index); +CREATE INDEX IF NOT EXISTS idx_chunks_page ON content_chunks(page_id); +CREATE INDEX IF NOT EXISTS idx_chunks_embedding ON content_chunks USING hnsw (embedding vector_cosine_ops); + +-- ============================================================ +-- links: cross-references between pages +-- ============================================================ +CREATE TABLE IF NOT EXISTS links ( + id SERIAL PRIMARY KEY, + from_page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE, + to_page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE, + link_type TEXT NOT NULL DEFAULT '', + context TEXT NOT NULL DEFAULT '', + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE(from_page_id, to_page_id) +); + +CREATE INDEX IF NOT EXISTS idx_links_from ON links(from_page_id); +CREATE INDEX IF NOT EXISTS idx_links_to ON links(to_page_id); + +-- ============================================================ +-- tags +-- ============================================================ +CREATE TABLE IF NOT EXISTS tags ( + id SERIAL PRIMARY KEY, + page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE, + tag TEXT NOT NULL, + UNIQUE(page_id, tag) +); + +CREATE INDEX IF NOT EXISTS idx_tags_tag ON tags(tag); +CREATE INDEX IF NOT EXISTS idx_tags_page_id ON tags(page_id); + +-- ============================================================ +-- raw_data: sidecar data (replaces .raw/ JSON files) +-- ============================================================ +CREATE TABLE IF NOT EXISTS raw_data ( + id SERIAL PRIMARY KEY, + page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE, + source TEXT NOT NULL, + data JSONB NOT NULL, + fetched_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE(page_id, source) +); + +CREATE INDEX IF NOT EXISTS idx_raw_data_page ON raw_data(page_id); + +-- ============================================================ +-- timeline_entries: structured timeline +-- ============================================================ +CREATE TABLE IF NOT EXISTS timeline_entries ( + id SERIAL PRIMARY KEY, + page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE, + date DATE NOT NULL, + source TEXT NOT NULL DEFAULT '', + summary TEXT NOT NULL, + detail TEXT NOT NULL DEFAULT '', + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_timeline_page ON timeline_entries(page_id); +CREATE INDEX IF NOT EXISTS idx_timeline_date ON timeline_entries(date); + +-- ============================================================ +-- page_versions: snapshot history for compiled_truth +-- ============================================================ +CREATE TABLE IF NOT EXISTS page_versions ( + id SERIAL PRIMARY KEY, + page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE, + compiled_truth TEXT NOT NULL, + frontmatter JSONB NOT NULL DEFAULT '{}', + snapshot_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_versions_page ON page_versions(page_id); + +-- ============================================================ +-- ingest_log +-- ============================================================ +CREATE TABLE IF NOT EXISTS ingest_log ( + id SERIAL PRIMARY KEY, + source_type TEXT NOT NULL, + source_ref TEXT NOT NULL, + pages_updated JSONB NOT NULL DEFAULT '[]', + summary TEXT NOT NULL DEFAULT '', + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- ============================================================ +-- config: brain-level settings +-- ============================================================ +CREATE TABLE IF NOT EXISTS config ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); + +INSERT INTO config (key, value) VALUES + ('version', '1'), + ('embedding_model', 'text-embedding-3-large'), + ('embedding_dimensions', '1536'), + ('chunk_strategy', 'semantic') +ON CONFLICT (key) DO NOTHING; + +-- ============================================================ +-- files: binary attachments stored in Supabase Storage +-- ============================================================ +CREATE TABLE IF NOT EXISTS files ( + id SERIAL PRIMARY KEY, + page_slug TEXT REFERENCES pages(slug) ON DELETE SET NULL ON UPDATE CASCADE, + filename TEXT NOT NULL, + storage_path TEXT NOT NULL, + mime_type TEXT, + size_bytes BIGINT, + content_hash TEXT NOT NULL, + metadata JSONB NOT NULL DEFAULT '{}', + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE(storage_path) +); + +-- Migration: drop storage_url if it exists (renamed to storage_path only) +ALTER TABLE files DROP COLUMN IF EXISTS storage_url; + +CREATE INDEX IF NOT EXISTS idx_files_page ON files(page_slug); +CREATE INDEX IF NOT EXISTS idx_files_hash ON files(content_hash); + +-- ============================================================ +-- Trigger-based search_vector (spans pages + timeline_entries) +-- ============================================================ +ALTER TABLE pages ADD COLUMN IF NOT EXISTS search_vector tsvector; + +CREATE INDEX IF NOT EXISTS idx_pages_search ON pages USING GIN(search_vector); + +-- Function to rebuild search_vector for a page +CREATE OR REPLACE FUNCTION update_page_search_vector() RETURNS trigger AS \$\$ +DECLARE + timeline_text TEXT; +BEGIN + -- Gather timeline_entries text for this page + SELECT coalesce(string_agg(summary || ' ' || detail, ' '), '') + INTO timeline_text + FROM timeline_entries + WHERE page_id = NEW.id; + + -- Build weighted tsvector + NEW.search_vector := + setweight(to_tsvector('english', coalesce(NEW.title, '')), 'A') || + setweight(to_tsvector('english', coalesce(NEW.compiled_truth, '')), 'B') || + setweight(to_tsvector('english', coalesce(NEW.timeline, '')), 'C') || + setweight(to_tsvector('english', coalesce(timeline_text, '')), 'C'); + + RETURN NEW; +END; +\$\$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS trg_pages_search_vector ON pages; +CREATE TRIGGER trg_pages_search_vector + BEFORE INSERT OR UPDATE ON pages + FOR EACH ROW + EXECUTE FUNCTION update_page_search_vector(); + +-- When timeline_entries change, update the parent page's search_vector +CREATE OR REPLACE FUNCTION update_page_search_vector_from_timeline() RETURNS trigger AS \$\$ +DECLARE + page_row pages%ROWTYPE; +BEGIN + -- Touch the page to re-fire its trigger + UPDATE pages SET updated_at = now() + WHERE id = coalesce(NEW.page_id, OLD.page_id); + RETURN NEW; +END; +\$\$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS trg_timeline_search_vector ON timeline_entries; +CREATE TRIGGER trg_timeline_search_vector + AFTER INSERT OR UPDATE OR DELETE ON timeline_entries + FOR EACH ROW + EXECUTE FUNCTION update_page_search_vector_from_timeline(); + +-- ============================================================ +-- Row Level Security: block anon access, postgres role bypasses +-- ============================================================ +-- The postgres role (used by gbrain via pooler) has BYPASSRLS. +-- Enabling RLS with no policies means the anon key can't read anything. +-- Only enable if the current role actually has BYPASSRLS privilege, +-- otherwise we'd lock ourselves out. +DO \$\$ +DECLARE + has_bypass BOOLEAN; +BEGIN + SELECT rolbypassrls INTO has_bypass FROM pg_roles WHERE rolname = current_user; + IF has_bypass THEN + ALTER TABLE pages ENABLE ROW LEVEL SECURITY; + ALTER TABLE content_chunks ENABLE ROW LEVEL SECURITY; + ALTER TABLE links ENABLE ROW LEVEL SECURITY; + ALTER TABLE tags ENABLE ROW LEVEL SECURITY; + ALTER TABLE raw_data ENABLE ROW LEVEL SECURITY; + ALTER TABLE timeline_entries ENABLE ROW LEVEL SECURITY; + ALTER TABLE page_versions ENABLE ROW LEVEL SECURITY; + ALTER TABLE ingest_log ENABLE ROW LEVEL SECURITY; + ALTER TABLE config ENABLE ROW LEVEL SECURITY; + ALTER TABLE files ENABLE ROW LEVEL SECURITY; + RAISE NOTICE 'RLS enabled on all tables (role % has BYPASSRLS)', current_user; + ELSE + RAISE WARNING 'Skipping RLS: role % does not have BYPASSRLS privilege. Run as postgres role to enable.', current_user; + END IF; +END \$\$; +`;