From ff282e96f121459a2288cf6afdfa0f55aea212aa Mon Sep 17 00:00:00 2001 From: jbarol Date: Wed, 10 Jun 2026 14:49:24 -0700 Subject: [PATCH] =?UTF-8?q?feat(migrate):=20v116=20=E2=80=94=20backfill=20?= =?UTF-8?q?NULL=20edge=20source=5Fid=20+=20index=20from=5Fsymbol=5Fqualifi?= =?UTF-8?q?ed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Backfill: edges written before the stamping fix sit at source_id=NULL and stay invisible to scoped call-graph queries until repaired. Derive each edge's source from its own from_chunk's page (pages.source_id is NOT NULL DEFAULT 'default'). Same SQL verified live on a 2,122-edge production brain. 2. Indexes: getCalleesOf filters both edge tables on from_symbol_qualified, which had no index — every callee lookup was a seq scan, amplified per-BFS-node by the recursive code walk. With NULL edges repaired, scoped walks actually expand, so the latent cost becomes real. Mirrored into src/schema.sql; schema-embedded.ts regenerated. --- src/core/migrate.ts | 46 +++++++++++++++++++++++++++++ src/core/schema-embedded.ts | 58 ++++++++++++++++++++++++++----------- src/schema.sql | 9 ++++++ 3 files changed, 96 insertions(+), 17 deletions(-) diff --git a/src/core/migrate.ts b/src/core/migrate.ts index 2e31e60ad..25e24df67 100644 --- a/src/core/migrate.ts +++ b/src/core/migrate.ts @@ -5186,6 +5186,52 @@ export const MIGRATIONS: Migration[] = [ ); `, }, + { + version: 116, + name: 'code_edges_source_backfill_and_callee_index', + // Repair + index pass for the call graph: + // + // 1. BACKFILL: importCodeFile built CodeEdgeInput rows without source_id, + // so every extracted edge landed NULL. getCallersOf/getCalleesOf add + // `AND source_id = ` whenever a worktree pin / --source is in + // play — NULL never matches, so scoped call-graph queries silently + // returned 0 rows on multi-source brains even though the edges + // existed. The write path now stamps `sourceId ?? 'default'`; this + // backfill repairs rows written before the fix by deriving each + // edge's source from its own from_chunk's page (pages.source_id is + // NOT NULL DEFAULT 'default', so COALESCE is belt-and-braces only). + // + // 2. INDEXES: getCalleesOf filters BOTH edge tables on + // from_symbol_qualified, which had no index anywhere — every callee + // lookup was a sequential scan, amplified per-BFS-node by the + // recursive code walk (one getCalleesOf per frontier node, up to + // maxNodes). With NULL edges repaired, scoped walks actually expand, + // so the latent seq-scan cost becomes real. Plain CREATE INDEX (not + // CONCURRENTLY): edge tables are modest (mirrors the v58 resolver + // index). Keep in sync with src/schema.sql. + idempotent: true, + sql: ` + UPDATE code_edges_symbol e + SET source_id = COALESCE(p.source_id, 'default') + FROM content_chunks c + JOIN pages p ON p.id = c.page_id + WHERE c.id = e.from_chunk_id + AND e.source_id IS NULL; + + UPDATE code_edges_chunk e + SET source_id = COALESCE(p.source_id, 'default') + FROM content_chunks c + JOIN pages p ON p.id = c.page_id + WHERE c.id = e.from_chunk_id + AND e.source_id IS NULL; + + CREATE INDEX IF NOT EXISTS idx_code_edges_symbol_from_symbol + ON code_edges_symbol (from_symbol_qualified); + + CREATE INDEX IF NOT EXISTS idx_code_edges_chunk_from_symbol + ON code_edges_chunk (from_symbol_qualified); + `, + }, ]; export const LATEST_VERSION = MIGRATIONS.length > 0 diff --git a/src/core/schema-embedded.ts b/src/core/schema-embedded.ts index dc288eabd..2e30b49f1 100644 --- a/src/core/schema-embedded.ts +++ b/src/core/schema-embedded.ts @@ -390,6 +390,11 @@ CREATE INDEX IF NOT EXISTS idx_code_edges_chunk_to ON code_edges_chunk(to_chunk_id, edge_type); CREATE INDEX IF NOT EXISTS idx_code_edges_chunk_to_symbol ON code_edges_chunk(to_symbol_qualified, edge_type); +-- getCalleesOf filters on from_symbol_qualified; without this index every +-- callee lookup is a sequential scan, amplified per-BFS-node by the +-- recursive code walk. Mirrors migration v116. +CREATE INDEX IF NOT EXISTS idx_code_edges_chunk_from_symbol + ON code_edges_chunk(from_symbol_qualified); CREATE TABLE IF NOT EXISTS code_edges_symbol ( id SERIAL PRIMARY KEY, @@ -407,16 +412,32 @@ CREATE INDEX IF NOT EXISTS idx_code_edges_symbol_from ON code_edges_symbol(from_chunk_id, edge_type); CREATE INDEX IF NOT EXISTS idx_code_edges_symbol_to ON code_edges_symbol(to_symbol_qualified, edge_type); +-- getCalleesOf companion to idx_code_edges_chunk_from_symbol above. +-- Mirrors migration v116. +CREATE INDEX IF NOT EXISTS idx_code_edges_symbol_from_symbol + ON code_edges_symbol(from_symbol_qualified); -- ============================================================ -- links: cross-references between pages -- ============================================================ --- Provenance model (v0.13, extended issue #972): --- link_source — 'markdown' | 'frontmatter' | 'manual' | 'wikilink-resolved' | NULL --- 'wikilink-resolved' is the opt-in --- (link_resolution.global_basename) basename-match --- provenance — see issue #972 / migration v113. --- (NULL = legacy row written before v0.13; unknown source) +-- Provenance model (v0.13; opened to kebab provenance in v114 / issue #1941): +-- link_source — open kebab-case provenance tag, NOT a closed allowlist. +-- Format gate (CHECK): ^[a-z][a-z0-9]*(-[a-z0-9]+)*\$ and +-- char_length <= 64. NULL = legacy row (pre-v0.13). +-- Reconciliation-managed built-ins written internally: +-- 'markdown' — body markdown links +-- 'frontmatter' — YAML frontmatter edges (see origin_*) +-- 'mentions' — auto-linked body-text mentions +-- 'wikilink-resolved'— opt-in global-basename [[name]] (#972) +-- User/tool-facing: +-- 'manual' — hand- or tool-created edges (the +-- add_link op default + CLI link-add) +-- '' — external derivers, e.g. 'citation-graph', +-- stamp their own kebab tag (no migration). +-- The add_link OP forbids callers from passing the four +-- managed built-ins (they imply reconciliation semantics a +-- hand-created row can't honor); the DB CHECK still admits +-- them because internal writers use them. See operations.ts. -- origin_page_id — for link_source='frontmatter', the page whose YAML -- frontmatter created this edge; scopes reconciliation -- origin_field — the frontmatter field name (e.g. 'key_people') @@ -424,21 +445,21 @@ CREATE INDEX IF NOT EXISTS idx_code_edges_symbol_to -- The unique constraint includes link_source + origin_page_id so a manual edge -- and a frontmatter-derived edge with the same (from, to, type) tuple coexist. -- Reconciliation on put_page filters by (link_source='frontmatter' AND --- origin_page_id = written_page) — never touches other pages' edges. +-- origin_page_id = written_page) — never touches other pages' edges. (This is +-- exactly why a CLI-forged 'frontmatter' row with NULL origin would be a phantom +-- edge reconciliation never cleans — hence the op-layer guard.) CREATE TABLE IF NOT EXISTS links ( id SERIAL PRIMARY KEY, from_page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE, to_page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE, link_type TEXT NOT NULL DEFAULT '', context TEXT NOT NULL DEFAULT '', - -- v114 (#1941): link_source is an open kebab-case provenance, not a closed - -- allowlist — external derivers (e.g. 'citation-graph') stamp their own tag - -- without a gbrain migration. Format gate only: lowercase kebab, <=64 chars. - -- The reconciliation-managed built-ins ('markdown','frontmatter','mentions', - -- 'wikilink-resolved') still satisfy this and are written internally; the - -- add_link op forbids CALLERS from forging them (see operations.ts). 'manual' - -- is the user-facing default for hand/tool-created edges. - link_source TEXT CHECK (link_source IS NULL OR (link_source ~ '^[a-z][a-z0-9]*(-[a-z0-9]+)*$' AND char_length(link_source) <= 64)), + -- v0.41.18.0: 'mentions' added for auto-linked body-text mentions + -- (gbrain extract links --by-mention). Filtered OUT of backlink-count + -- for search ranking; only counts toward orphan-ratio + graph traversal. + -- v0.40.8.2 (#972): 'wikilink-resolved' added for opt-in global-basename + -- wikilink resolution (bare [[name]] resolved by slug tail). + link_source TEXT CHECK (link_source IS NULL OR (link_source ~ '^[a-z][a-z0-9]*(-[a-z0-9]+)*\$' AND char_length(link_source) <= 64)), -- v0.41.18.0: nullable link_kind distinguishes "plain body mention" from -- "verb-pattern-derived typed link" within link_source='mentions'. -- Codex finding #12 design: keep link_source stable; add link_kind @@ -675,8 +696,11 @@ CREATE TABLE IF NOT EXISTS op_checkpoints ( CREATE INDEX IF NOT EXISTS op_checkpoints_updated_at_idx ON op_checkpoints (updated_at); --- #1794: append-only delta storage (one row per completed path). FK cascade --- drops children with the parent. Mirrors migration v115 + src/schema.sql. +-- #1794: append-only delta storage. One row per completed path; sync's +-- appendCompleted INSERTs only the delta instead of rewriting the whole +-- completed_keys JSONB array each flush (O(N^2) -> O(delta)). FK cascade drops +-- children with the parent (clearOpCheckpoint + 7-day purge). PK prefix +-- (op,fingerprint) serves all reads; no separate index. Mirrors migration v115. CREATE TABLE IF NOT EXISTS op_checkpoint_paths ( op TEXT NOT NULL, fingerprint TEXT NOT NULL, diff --git a/src/schema.sql b/src/schema.sql index a900c6d1a..3a5f32ef7 100644 --- a/src/schema.sql +++ b/src/schema.sql @@ -386,6 +386,11 @@ CREATE INDEX IF NOT EXISTS idx_code_edges_chunk_to ON code_edges_chunk(to_chunk_id, edge_type); CREATE INDEX IF NOT EXISTS idx_code_edges_chunk_to_symbol ON code_edges_chunk(to_symbol_qualified, edge_type); +-- getCalleesOf filters on from_symbol_qualified; without this index every +-- callee lookup is a sequential scan, amplified per-BFS-node by the +-- recursive code walk. Mirrors migration v116. +CREATE INDEX IF NOT EXISTS idx_code_edges_chunk_from_symbol + ON code_edges_chunk(from_symbol_qualified); CREATE TABLE IF NOT EXISTS code_edges_symbol ( id SERIAL PRIMARY KEY, @@ -403,6 +408,10 @@ CREATE INDEX IF NOT EXISTS idx_code_edges_symbol_from ON code_edges_symbol(from_chunk_id, edge_type); CREATE INDEX IF NOT EXISTS idx_code_edges_symbol_to ON code_edges_symbol(to_symbol_qualified, edge_type); +-- getCalleesOf companion to idx_code_edges_chunk_from_symbol above. +-- Mirrors migration v116. +CREATE INDEX IF NOT EXISTS idx_code_edges_symbol_from_symbol + ON code_edges_symbol(from_symbol_qualified); -- ============================================================ -- links: cross-references between pages