mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
fix: embed schema.sql at build time, remove fs dependency from initSchema
initSchema() previously read schema.sql from disk at runtime via readFileSync, which broke in compiled Bun binaries and Deno Edge Functions. Now uses a generated schema-embedded.ts constant (run `bun run build:schema` to regenerate). - Removes fs and path imports from postgres-engine.ts and db.ts - Adds scripts/build-schema.sh for one-source-of-truth generation - Adds build:schema npm script Fixes Issue #22 Bug #6. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
co-authored by
Claude Opus 4.6
Claude Opus 4.6
parent
a464406338
commit
cae6b0c97a
@@ -17,6 +17,7 @@
|
||||
"dev": "bun run src/cli.ts",
|
||||
"build": "bun build --compile --outfile bin/gbrain src/cli.ts",
|
||||
"build:all": "bun build --compile --target=bun-darwin-arm64 --outfile bin/gbrain-darwin-arm64 src/cli.ts && bun build --compile --target=bun-linux-x64 --outfile bin/gbrain-linux-x64 src/cli.ts",
|
||||
"build:schema": "bash scripts/build-schema.sh",
|
||||
"test": "bun test",
|
||||
"test:e2e": "bun test test/e2e/",
|
||||
"prepublish:clawhub": "bun run build:all",
|
||||
|
||||
Executable
+15
@@ -0,0 +1,15 @@
|
||||
#!/bin/bash
|
||||
# Generate src/core/schema-embedded.ts from src/schema.sql
|
||||
# One source of truth: schema.sql is the canonical file.
|
||||
# This script produces a TypeScript constant for use in compiled binaries and Edge Functions.
|
||||
set -e
|
||||
SCHEMA_FILE="src/schema.sql"
|
||||
OUT_FILE="src/core/schema-embedded.ts"
|
||||
echo "// AUTO-GENERATED — do not edit. Run: bun run build:schema" > "$OUT_FILE"
|
||||
echo "// Source: $SCHEMA_FILE" >> "$OUT_FILE"
|
||||
echo "" >> "$OUT_FILE"
|
||||
echo "export const SCHEMA_SQL = \`" >> "$OUT_FILE"
|
||||
# Escape backticks and dollar signs in the SQL for template literal safety
|
||||
sed 's/`/\\`/g; s/\$/\\$/g' "$SCHEMA_FILE" >> "$OUT_FILE"
|
||||
echo "\`;" >> "$OUT_FILE"
|
||||
echo "Generated $OUT_FILE from $SCHEMA_FILE"
|
||||
+2
-11
@@ -1,7 +1,6 @@
|
||||
import postgres from 'postgres';
|
||||
import { readFileSync } from 'fs';
|
||||
import { join, dirname } from 'path';
|
||||
import { GBrainError, type EngineConfig } from './types.ts';
|
||||
import { SCHEMA_SQL } from './schema-embedded.ts';
|
||||
|
||||
let sql: ReturnType<typeof postgres> | null = null;
|
||||
|
||||
@@ -61,15 +60,7 @@ export async function disconnect(): Promise<void> {
|
||||
|
||||
export async function initSchema(): Promise<void> {
|
||||
const conn = getConnection();
|
||||
|
||||
// Read schema SQL and execute as a single statement.
|
||||
// The postgres driver handles multi-statement SQL natively, including
|
||||
// PL/pgSQL functions with $$ delimiter blocks that contain semicolons.
|
||||
// The schema uses IF NOT EXISTS / CREATE OR REPLACE for idempotency.
|
||||
const schemaPath = join(dirname(new URL(import.meta.url).pathname), '..', 'schema.sql');
|
||||
const schemaSql = readFileSync(schemaPath, 'utf-8');
|
||||
|
||||
await conn.unsafe(schemaSql);
|
||||
await conn.unsafe(SCHEMA_SQL);
|
||||
}
|
||||
|
||||
export async function withTransaction<T>(fn: (tx: ReturnType<typeof postgres>) => Promise<T>): Promise<T> {
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
import postgres from 'postgres';
|
||||
import { createHash } from 'crypto';
|
||||
import { readFileSync } from 'fs';
|
||||
import { join, dirname } from 'path';
|
||||
import type { BrainEngine } from './engine.ts';
|
||||
import { runMigrations } from './migrate.ts';
|
||||
import { SCHEMA_SQL } from './schema-embedded.ts';
|
||||
import type {
|
||||
Page, PageInput, PageFilters, PageType,
|
||||
Chunk, ChunkInput,
|
||||
@@ -58,9 +57,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
|
||||
async initSchema(): Promise<void> {
|
||||
const conn = this.sql;
|
||||
const schemaPath = join(dirname(new URL(import.meta.url).pathname), '..', 'schema.sql');
|
||||
const schemaSql = readFileSync(schemaPath, 'utf-8');
|
||||
await conn.unsafe(schemaSql);
|
||||
await conn.unsafe(SCHEMA_SQL);
|
||||
|
||||
// Run any pending migrations automatically
|
||||
const { applied } = await runMigrations(this);
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
// AUTO-GENERATED — do not edit. Run: bun run build:schema
|
||||
// Source: src/schema.sql
|
||||
|
||||
export const SCHEMA_SQL = `
|
||||
-- GBrain Postgres + pgvector schema
|
||||
|
||||
CREATE EXTENSION IF NOT EXISTS vector;
|
||||
CREATE EXTENSION IF NOT EXISTS pg_trgm;
|
||||
|
||||
-- ============================================================
|
||||
-- pages: the core content table
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS pages (
|
||||
id SERIAL PRIMARY KEY,
|
||||
slug TEXT NOT NULL UNIQUE,
|
||||
type TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
compiled_truth TEXT NOT NULL DEFAULT '',
|
||||
timeline TEXT NOT NULL DEFAULT '',
|
||||
frontmatter JSONB NOT NULL DEFAULT '{}',
|
||||
content_hash TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_pages_type ON pages(type);
|
||||
CREATE INDEX IF NOT EXISTS idx_pages_frontmatter ON pages USING GIN(frontmatter);
|
||||
CREATE INDEX IF NOT EXISTS idx_pages_trgm ON pages USING GIN(title gin_trgm_ops);
|
||||
|
||||
-- ============================================================
|
||||
-- content_chunks: chunked content with embeddings
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS content_chunks (
|
||||
id SERIAL PRIMARY KEY,
|
||||
page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
|
||||
chunk_index INTEGER NOT NULL,
|
||||
chunk_text TEXT NOT NULL,
|
||||
chunk_source TEXT NOT NULL DEFAULT 'compiled_truth',
|
||||
embedding vector(1536),
|
||||
model TEXT NOT NULL DEFAULT 'text-embedding-3-large',
|
||||
token_count INTEGER,
|
||||
embedded_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_chunks_page_index ON content_chunks(page_id, chunk_index);
|
||||
CREATE INDEX IF NOT EXISTS idx_chunks_page ON content_chunks(page_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_chunks_embedding ON content_chunks USING hnsw (embedding vector_cosine_ops);
|
||||
|
||||
-- ============================================================
|
||||
-- links: cross-references between pages
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS links (
|
||||
id SERIAL PRIMARY KEY,
|
||||
from_page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
|
||||
to_page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
|
||||
link_type TEXT NOT NULL DEFAULT '',
|
||||
context TEXT NOT NULL DEFAULT '',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE(from_page_id, to_page_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_links_from ON links(from_page_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_links_to ON links(to_page_id);
|
||||
|
||||
-- ============================================================
|
||||
-- tags
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS tags (
|
||||
id SERIAL PRIMARY KEY,
|
||||
page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
|
||||
tag TEXT NOT NULL,
|
||||
UNIQUE(page_id, tag)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_tags_tag ON tags(tag);
|
||||
CREATE INDEX IF NOT EXISTS idx_tags_page_id ON tags(page_id);
|
||||
|
||||
-- ============================================================
|
||||
-- raw_data: sidecar data (replaces .raw/ JSON files)
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS raw_data (
|
||||
id SERIAL PRIMARY KEY,
|
||||
page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
|
||||
source TEXT NOT NULL,
|
||||
data JSONB NOT NULL,
|
||||
fetched_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE(page_id, source)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_data_page ON raw_data(page_id);
|
||||
|
||||
-- ============================================================
|
||||
-- timeline_entries: structured timeline
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS timeline_entries (
|
||||
id SERIAL PRIMARY KEY,
|
||||
page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
|
||||
date DATE NOT NULL,
|
||||
source TEXT NOT NULL DEFAULT '',
|
||||
summary TEXT NOT NULL,
|
||||
detail TEXT NOT NULL DEFAULT '',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_timeline_page ON timeline_entries(page_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_timeline_date ON timeline_entries(date);
|
||||
|
||||
-- ============================================================
|
||||
-- page_versions: snapshot history for compiled_truth
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS page_versions (
|
||||
id SERIAL PRIMARY KEY,
|
||||
page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
|
||||
compiled_truth TEXT NOT NULL,
|
||||
frontmatter JSONB NOT NULL DEFAULT '{}',
|
||||
snapshot_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_versions_page ON page_versions(page_id);
|
||||
|
||||
-- ============================================================
|
||||
-- ingest_log
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS ingest_log (
|
||||
id SERIAL PRIMARY KEY,
|
||||
source_type TEXT NOT NULL,
|
||||
source_ref TEXT NOT NULL,
|
||||
pages_updated JSONB NOT NULL DEFAULT '[]',
|
||||
summary TEXT NOT NULL DEFAULT '',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
-- ============================================================
|
||||
-- config: brain-level settings
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS config (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
|
||||
INSERT INTO config (key, value) VALUES
|
||||
('version', '1'),
|
||||
('embedding_model', 'text-embedding-3-large'),
|
||||
('embedding_dimensions', '1536'),
|
||||
('chunk_strategy', 'semantic')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
|
||||
-- ============================================================
|
||||
-- files: binary attachments stored in Supabase Storage
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS files (
|
||||
id SERIAL PRIMARY KEY,
|
||||
page_slug TEXT REFERENCES pages(slug) ON DELETE SET NULL ON UPDATE CASCADE,
|
||||
filename TEXT NOT NULL,
|
||||
storage_path TEXT NOT NULL,
|
||||
mime_type TEXT,
|
||||
size_bytes BIGINT,
|
||||
content_hash TEXT NOT NULL,
|
||||
metadata JSONB NOT NULL DEFAULT '{}',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE(storage_path)
|
||||
);
|
||||
|
||||
-- Migration: drop storage_url if it exists (renamed to storage_path only)
|
||||
ALTER TABLE files DROP COLUMN IF EXISTS storage_url;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_files_page ON files(page_slug);
|
||||
CREATE INDEX IF NOT EXISTS idx_files_hash ON files(content_hash);
|
||||
|
||||
-- ============================================================
|
||||
-- Trigger-based search_vector (spans pages + timeline_entries)
|
||||
-- ============================================================
|
||||
ALTER TABLE pages ADD COLUMN IF NOT EXISTS search_vector tsvector;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_pages_search ON pages USING GIN(search_vector);
|
||||
|
||||
-- Function to rebuild search_vector for a page
|
||||
CREATE OR REPLACE FUNCTION update_page_search_vector() RETURNS trigger AS \$\$
|
||||
DECLARE
|
||||
timeline_text TEXT;
|
||||
BEGIN
|
||||
-- Gather timeline_entries text for this page
|
||||
SELECT coalesce(string_agg(summary || ' ' || detail, ' '), '')
|
||||
INTO timeline_text
|
||||
FROM timeline_entries
|
||||
WHERE page_id = NEW.id;
|
||||
|
||||
-- Build weighted tsvector
|
||||
NEW.search_vector :=
|
||||
setweight(to_tsvector('english', coalesce(NEW.title, '')), 'A') ||
|
||||
setweight(to_tsvector('english', coalesce(NEW.compiled_truth, '')), 'B') ||
|
||||
setweight(to_tsvector('english', coalesce(NEW.timeline, '')), 'C') ||
|
||||
setweight(to_tsvector('english', coalesce(timeline_text, '')), 'C');
|
||||
|
||||
RETURN NEW;
|
||||
END;
|
||||
\$\$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS trg_pages_search_vector ON pages;
|
||||
CREATE TRIGGER trg_pages_search_vector
|
||||
BEFORE INSERT OR UPDATE ON pages
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_page_search_vector();
|
||||
|
||||
-- When timeline_entries change, update the parent page's search_vector
|
||||
CREATE OR REPLACE FUNCTION update_page_search_vector_from_timeline() RETURNS trigger AS \$\$
|
||||
DECLARE
|
||||
page_row pages%ROWTYPE;
|
||||
BEGIN
|
||||
-- Touch the page to re-fire its trigger
|
||||
UPDATE pages SET updated_at = now()
|
||||
WHERE id = coalesce(NEW.page_id, OLD.page_id);
|
||||
RETURN NEW;
|
||||
END;
|
||||
\$\$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS trg_timeline_search_vector ON timeline_entries;
|
||||
CREATE TRIGGER trg_timeline_search_vector
|
||||
AFTER INSERT OR UPDATE OR DELETE ON timeline_entries
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_page_search_vector_from_timeline();
|
||||
|
||||
-- ============================================================
|
||||
-- Row Level Security: block anon access, postgres role bypasses
|
||||
-- ============================================================
|
||||
-- The postgres role (used by gbrain via pooler) has BYPASSRLS.
|
||||
-- Enabling RLS with no policies means the anon key can't read anything.
|
||||
-- Only enable if the current role actually has BYPASSRLS privilege,
|
||||
-- otherwise we'd lock ourselves out.
|
||||
DO \$\$
|
||||
DECLARE
|
||||
has_bypass BOOLEAN;
|
||||
BEGIN
|
||||
SELECT rolbypassrls INTO has_bypass FROM pg_roles WHERE rolname = current_user;
|
||||
IF has_bypass THEN
|
||||
ALTER TABLE pages ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE content_chunks ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE links ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE tags ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE raw_data ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE timeline_entries ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE page_versions ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE ingest_log ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE config ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE files ENABLE ROW LEVEL SECURITY;
|
||||
RAISE NOTICE 'RLS enabled on all tables (role % has BYPASSRLS)', current_user;
|
||||
ELSE
|
||||
RAISE WARNING 'Skipping RLS: role % does not have BYPASSRLS privilege. Run as postgres role to enable.', current_user;
|
||||
END IF;
|
||||
END \$\$;
|
||||
`;
|
||||
Reference in New Issue
Block a user