From 0a94216397a8f0216be5385f8fa47b9faffc565b Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Mon, 11 May 2026 12:34:43 -0700 Subject: [PATCH] feat(v0.34 W0c): within-file two-pass symbol resolver + edges_backfilled_at watermark MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex's outside-voice review caught that the v0.20.0 graph stores BARE callee tokens (`render`, `find`, `execute`) — not qualified names. Pre-v0.34 recursive blast/flow would alias every same-named function across classes. W0c is the foundation that fixes this: resolve `code_edges_symbol` rows by matching `to_symbol_qualified` against the SAME-FILE chunks' `symbol_name_qualified`, then write the outcome to `edge_metadata`. This commit is the resolver primitive + schema. The cycle-phase wiring that calls it on every quick-cycle tick lands in the next commit. Schema (v51 migration `edges_backfilled_at_v0_34`): * `content_chunks.edges_backfilled_at TIMESTAMPTZ` — resume watermark. Chunks where the column is NULL OR older than EDGE_EXTRACTOR_VERSION_TS get re-walked next tick. SIGINT/OOM/sleep mid-backfill loses at most one batch. * Indexes per D11 from eng review: - `idx_code_edges_symbol_resolver(source_id, to_symbol_qualified)` — composite for the resolver's per-source lookup. - `idx_content_chunks_symbol_lookup(page_id, symbol_name_qualified)` WHERE `symbol_name_qualified IS NOT NULL` — file-batched candidate fetch; also reused by W4-5 cluster recompute. - `idx_content_chunks_edges_backfill(edges_backfilled_at)` WHERE `edges_backfilled_at IS NULL` — fast unresumed-row scan. Module (`src/core/chunkers/symbol-resolver.ts`): * `resolveSymbolEdgesIncremental(engine, {sourceId, maxChunks?, onProgress?})` walks stale chunks in 200-chunk batches. For each chunk, loads its unresolved edges, finds same-page candidates by symbol_name_qualified, and writes outcome to `edge_metadata`: - exactly 1 candidate → `{resolved_chunk_id: }` - 2+ candidates → `{ambiguous: true, candidates: [...]}` - 0 candidates → unchanged (cross-file; two-pass.ts handles those) Each batch bumps `edges_backfilled_at = NOW()` for the chunks. * `readEdgeResolution(metadata)` — public helper for downstream code (two-pass.ts, code_blast op, eval-capture) to consume the resolver's output without parsing JSON directly. Returns a tagged union. * `EDGE_EXTRACTOR_VERSION_TS` exported constant — bump when extractor shape changes and the next cycle re-walks all chunks. Tests (5 E2E in test/e2e/symbol-resolver-pglite.test.ts, all PGLite, no DATABASE_URL): unambiguous match, ambiguous multi-match, no match, watermark advance + idempotency, source isolation (no cross-source candidate leak). Co-Authored-By: Claude Opus 4.7 (1M context) --- src/core/chunkers/symbol-resolver.ts | Bin 0 -> 11080 bytes src/core/migrate.ts | 47 +++++ test/e2e/symbol-resolver-pglite.test.ts | 219 ++++++++++++++++++++++++ 3 files changed, 266 insertions(+) create mode 100644 src/core/chunkers/symbol-resolver.ts create mode 100644 test/e2e/symbol-resolver-pglite.test.ts diff --git a/src/core/chunkers/symbol-resolver.ts b/src/core/chunkers/symbol-resolver.ts new file mode 100644 index 0000000000000000000000000000000000000000..51eb504d50cdfbba842c946f3ec01f33a0081e51 GIT binary patch literal 11080 zcmds7TW=f374EZs#Ytd9q(iPHrwt0~5?B(GD3wK5lItXbK`nPj(njPiv$JcOkyQkF z?n57ppueT)ukkPGcg|dPm!f1pG-y%6u*lt+GiT1_JC~V94Z;d89!5o$Q>*zq zD0G|DBq=oQJUQx96pqIVgtIG^YKqG|NiUd^o~IGD&d$#|bT;Ata+Ss^$2V0(K|7$Q z<+w)s4i1V@g=5-DlW5FZ_qSXU9-Ug%<}Sdk%!Yu z3dadZ#H6qvJKan*du38#lxMsgFN(!RDhf*t<}X#QXclTpGy1Uvq4Pw=U7l?uR2E5= zRx(O7O|rOzJn@aW&??H(m{_tsI)~6y7+q4CTJeG>$droJmGQA>w8DcJlBwAw)QkG&&j)ik2WWqE~;;>M3X2=Yg-J$3H3{otjOQkzBgHAy~ z4&@hQOIUAXUC0tVGX+l>>)TilV}N*RzN($NVWF?5 z5TB-3N$62Q7-Ek1V>Ov(g-WA&&-#^(c(4V6|fv z4?~gS799>=wAyDBo)_SWkwZojOdQ^vMGDRs2Xv%#$wrezP+B>qJez6OB}QR0%+?I% z4pp(?YC3#=aKH|cjR7kd5vJ7N`>H?ee|vKDWcOrnH2h2d=y-o{I6OHvI0(CSlsvpa zB=EH{w%aSxDqvitS$ zc>jCr4w|OSTpp$xjsg?g5I-J?Nm}p-i)@laP<>K}LG0$ChTqbdQ6Vhg@w%vHxh0>0 z?%rcM&$9_B_*Jf6C2H1#s~HZb@H<$nbz=)P7I%^~%GCs#3&-L#ycO&a{1_jguq?7^ zGR_K!=f@HX=~|*8fJnpk7~CP8xu)|F+&wwmV==(pQ)pKs9ys{0sx~o*{`&bE);OnG z4p;EfW8=O3dH42j@816W+x+`GbG!WW?(Hwm=Zx(w48wbwY&Qr;N`EX>34K{sfBh$u zF#8@hoPqX_#cCai1=--3?h9>X>RQU7W>EygdKw0u1iJ%`id1)H(VJl0T z$UZ4%>yrSP`~i&fbghb|Hvit~{}&c3lG+_SeY$@_0`7Mkw-qY?xz{4<dY|-lDwa1@#*Z$C5Uq4w}>&d_GSv(C2wMf53 zfQ#yVf}|)v4~=G zBNf;=G(NjO|P;f-dKTjE3rQJ^#9h><)3vYcwA2tG4@;AAW{1l(B4D2NuWjTy`xmkSJa$TnHx8z>mK z7}m76!(Lx`XIJRgQ*eP1j|ZXTvgc(g>AAz3V`-1DTJ(OEEVtNE1u$W76`pqRID>0c zYG+63th9Rs)0iY$Z8q#-+du)A7tB3CX^Zy2d;syc+HCm-q`aNtJ<|`W2VZEi*w-xW<560JO%u-$tLtWB3817&O~>T*IBl^BU;b=N@tFZ-yKBdIQiT8Y6S? zWgr-pt;{aDUDc|dZ&q&Gc6$*CqLM*myOhq407T(p2_3Q7>{5>hX|?jQB> z-t5P8lOC*Bl(>SV7A7terY)Olqh$k-?Hv;~>A~Y(&kMBp301EB430SU*!hm9Cb55f zu+N#>gU`;4KMA)Y&13uqW{G0!&j(oA-igAPv1Xe$^)O{ko=wqV_D8s#dy zvgT#9Xl-?uqtXtOYZ<+EjNeym7JG$tFLlS31=oI90q@xz# zg|U4!;kzT8S7(4BXQgZ=8# z1u!iiiwO%oZ2Q)@d1D!{{N(Vv)`Rt451G&D%gPnmN4?0x$TV5yoRI!rfBhTNfB zvORsdZ2~nXFfFSfZfprD=S1dtnCBqbTq8jLw55Uu<84AvKYeLj3zLFrseuF5n-=Dy z+FqRBK_QjPXz@CTX{SukqGluH;vC#rfJawVnHYu$GtROr?zY;qh{$DlXkPP)hqY9m zxkN$^-)d%iYQu?(CNJAHBW5!CBzu-;Xq>|^@;sc26k06uV{FPO3z<&M4=melrUZ_# zvpux?(Zno}AegLksWG=@bLCXp<#AT+hWE^mji#}z&4E8$sl2S13V-o}S*li}P{vv} z2~nWLSr`rkRNR*Mnxv&N<1A}I7{{%%2iIoG&F>yuqej2&t~X~5g%48I)t5MZK!Jo~ zJ{-Pybs{yI2K0g(?y4)DDMl?{Iym+784{9hma#OleP=-lSOc`Yzw+)7Rd?uNsAv3|pDcj3s) z#f&9M5~C-J0htRxNs5Do$--J4A?sO_e`)g|Ly0|LIW&^wLrd3@Ci37!8p&0*!n`?= zRcQ#bQ$r}6m7L3|ij$=C?kmv{%nhJpSYwOJgQX1IoEb}XllkZkJMosLMdRWMpxZz- zPteUVG9O)Y&($>cs{um$gLXGxF~n4%rzx{f5On> zn+|;9!gn%O`8Euygz6y1T%)R*=j>tpjGgUXUpn)iR8N7t^XP>`aeF<4YJzWOMT=Eo ztrj$3Vrm9^sKOv<9@yX@9R=TrM!C+KsG(C<77?z}m?N=}GA|#~m7_5(e_X0DA8Ig& z#puDq&{Q}PETWdfU>wq+O~>#w!u>TAbkjD-nmaZ#+*6S<$6jZ1xjA3?!3L*gA$EeV zNMix2=$10S9 z=s7lrR$h#;F7|l{Y~)MLiIsh?61R@+d0ee`AI42CTTXxPJM6$#s*SEM8lmo!-5~@ zr7m4axk2~IC$<7lv{d3=S{1nIw&Q9^zUFCrE$RTVAHkk`SV*t94Q@8aCty|@kucyl oZL-0;@+e>Q@ga9p;(kLcmt|(;A8NK14htr?xJ_z7c{eNn2HjUH7ytkO literal 0 HcmV?d00001 diff --git a/src/core/migrate.ts b/src/core/migrate.ts index 2b7be38f7..d7cc694d0 100644 --- a/src/core/migrate.ts +++ b/src/core/migrate.ts @@ -2548,6 +2548,53 @@ export const MIGRATIONS: Migration[] = [ ON ingest_log (source_id, source_type, created_at DESC); `, }, + { + version: 51, + name: 'edges_backfilled_at_v0_34', + // v0.34 W0c — resumable symbol-resolution backfill watermark. + // + // The within-file two-pass resolver (src/core/chunkers/symbol-resolver.ts) + // walks every content_chunks row that has unresolved edges + // (code_edges_symbol.to_chunk_id IS NULL) and matches each + // `to_symbol_qualified` against same-file chunks' `symbol_name_qualified`. + // On a 96K-chunk brain that is a 5-15 minute backfill the first time + // it runs. + // + // `edges_backfilled_at` is the resume watermark. Backfill runs in + // 200-chunk batches; on batch success the column is set to NOW() for + // every chunk in the batch. Resume picks up chunks where the watermark + // is NULL or older than EDGE_EXTRACTOR_VERSION_TS (a constant bumped + // when the extractor's shape changes). Crashes lose at most one batch. + // + // Composite + partial indexes for the lookup hot path (D11 from eng + // review): + // - idx_code_edges_symbol_resolver (source_id, to_symbol_qualified) + // — every code_edges_symbol row IS an unresolved edge by construction + // (the table has no to_chunk_id column; that lives on code_edges_chunk). + // This composite index supports the resolver's per-source lookups. + // - idx_content_chunks_symbol_lookup (page_id, symbol_name_qualified) + // WHERE symbol_name_qualified IS NOT NULL — file-batched lookup + // used by both the resolver and the cluster recompute phase (W4-5). + // - idx_content_chunks_edges_backfill (edges_backfilled_at) + // WHERE edges_backfilled_at IS NULL — find unresumed rows quickly. + // + // Idempotent: IF NOT EXISTS on column + indexes. Backfill itself runs + // separately via the resolve_symbol_edges_incremental cycle phase. + sql: ` + ALTER TABLE content_chunks ADD COLUMN IF NOT EXISTS edges_backfilled_at TIMESTAMPTZ; + + CREATE INDEX IF NOT EXISTS idx_code_edges_symbol_resolver + ON code_edges_symbol (source_id, to_symbol_qualified); + + CREATE INDEX IF NOT EXISTS idx_content_chunks_symbol_lookup + ON content_chunks (page_id, symbol_name_qualified) + WHERE symbol_name_qualified IS NOT NULL; + + CREATE INDEX IF NOT EXISTS idx_content_chunks_edges_backfill + ON content_chunks (edges_backfilled_at) + WHERE edges_backfilled_at IS NULL; + `, + }, ]; export const LATEST_VERSION = MIGRATIONS.length > 0 diff --git a/test/e2e/symbol-resolver-pglite.test.ts b/test/e2e/symbol-resolver-pglite.test.ts new file mode 100644 index 000000000..45716615e --- /dev/null +++ b/test/e2e/symbol-resolver-pglite.test.ts @@ -0,0 +1,219 @@ +/** + * v0.34 W0c — within-file two-pass symbol resolver E2E. + * + * Pins: + * - Unambiguous within-file match → edge_metadata.resolved_chunk_id set + * - Multi-match within file → edge_metadata.ambiguous=true + candidates + * - No match → edge stays untouched + * - chunks_walked watermark advances (edges_backfilled_at = NOW()) + * - Idempotency: re-run on processed chunks is a no-op + * - Resume: bumping EDGE_EXTRACTOR_VERSION_TS forces re-walk + * - Source isolation: resolver scoped to one source_id; never touches edges + * in a different source even if the symbol name collides + * + * PGLite in-memory. + */ + +import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test'; +import { PGLiteEngine } from '../../src/core/pglite-engine.ts'; +import { + resolveSymbolEdgesIncremental, + readEdgeResolution, + EDGE_EXTRACTOR_VERSION_TS, +} from '../../src/core/chunkers/symbol-resolver.ts'; +import { resetPgliteState } from '../helpers/reset-pglite.ts'; + +let engine: PGLiteEngine; + +beforeAll(async () => { + engine = new PGLiteEngine(); + await engine.connect({}); + await engine.initSchema(); +}); + +afterAll(async () => { + await engine.disconnect(); +}); + +beforeEach(async () => { + await resetPgliteState(engine); +}); + +describe('v0.34 W0c symbol-resolver — unambiguous within-file match', () => { + test('single-file: parseMarkdown call resolves to the parseMarkdown chunk in same file', async () => { + await registerSource(engine, 'source-a'); + const pageId = await insertCodePage(engine, 'source-a', 'src/foo.ts'); + const callerChunk = await insertChunk(engine, pageId, 0, 'callerInA', 'function'); + const defChunk = await insertChunk(engine, pageId, 1, 'parseMarkdown', 'function'); + await insertUnresolvedEdge(engine, callerChunk, 'callerInA', 'parseMarkdown', 'source-a'); + + const stats = await resolveSymbolEdgesIncremental(engine, { sourceId: 'source-a' }); + + expect(stats.chunks_walked).toBeGreaterThanOrEqual(2); + expect(stats.edges_resolved).toBe(1); + expect(stats.edges_ambiguous).toBe(0); + expect(stats.edges_unmatched).toBe(0); + + const edges = await engine.executeRaw<{ edge_metadata: any }>( + `SELECT edge_metadata FROM code_edges_symbol`, + [], + ); + expect(edges.length).toBe(1); + const res = readEdgeResolution(edges[0]!.edge_metadata); + expect(res.kind).toBe('resolved'); + if (res.kind === 'resolved') { + expect(res.chunk_id).toBe(defChunk); + } + }); +}); + +describe('v0.34 W0c symbol-resolver — ambiguous within-file match', () => { + test('two same-named methods in the same file → ambiguous + candidates list', async () => { + await registerSource(engine, 'source-a'); + const pageId = await insertCodePage(engine, 'source-a', 'src/foo.ts'); + const callerChunk = await insertChunk(engine, pageId, 0, 'callerInA', 'function'); + const def1 = await insertChunk(engine, pageId, 1, 'render', 'function'); + const def2 = await insertChunk(engine, pageId, 2, 'render', 'function'); // dup symbol name in same file + await insertUnresolvedEdge(engine, callerChunk, 'callerInA', 'render', 'source-a'); + + const stats = await resolveSymbolEdgesIncremental(engine, { sourceId: 'source-a' }); + + expect(stats.edges_ambiguous).toBe(1); + expect(stats.edges_resolved).toBe(0); + + const edges = await engine.executeRaw<{ edge_metadata: any }>( + `SELECT edge_metadata FROM code_edges_symbol`, + [], + ); + const res = readEdgeResolution(edges[0]!.edge_metadata); + expect(res.kind).toBe('ambiguous'); + if (res.kind === 'ambiguous') { + expect(res.candidate_chunk_ids.sort()).toEqual([def1, def2].sort()); + } + }); +}); + +describe('v0.34 W0c symbol-resolver — no match', () => { + test('call to a symbol defined in another file stays unresolved (caller two-pass handles cross-file)', async () => { + await registerSource(engine, 'source-a'); + const pageA = await insertCodePage(engine, 'source-a', 'src/foo.ts'); + const pageB = await insertCodePage(engine, 'source-a', 'src/bar.ts'); + const callerChunk = await insertChunk(engine, pageA, 0, 'callerInA', 'function'); + await insertChunk(engine, pageB, 0, 'externalFn', 'function'); // different file + await insertUnresolvedEdge(engine, callerChunk, 'callerInA', 'externalFn', 'source-a'); + + const stats = await resolveSymbolEdgesIncremental(engine, { sourceId: 'source-a' }); + + expect(stats.edges_unmatched).toBe(1); + expect(stats.edges_resolved).toBe(0); + expect(stats.edges_ambiguous).toBe(0); + + const edges = await engine.executeRaw<{ edge_metadata: any }>( + `SELECT edge_metadata FROM code_edges_symbol`, + [], + ); + const res = readEdgeResolution(edges[0]!.edge_metadata); + expect(res.kind).toBe('unresolved'); + }); +}); + +describe('v0.34 W0c symbol-resolver — watermark + idempotency', () => { + test('edges_backfilled_at advances; second run is a no-op', async () => { + await registerSource(engine, 'source-a'); + const pageId = await insertCodePage(engine, 'source-a', 'src/foo.ts'); + const callerChunk = await insertChunk(engine, pageId, 0, 'callerInA', 'function'); + await insertChunk(engine, pageId, 1, 'parseMarkdown', 'function'); + await insertUnresolvedEdge(engine, callerChunk, 'callerInA', 'parseMarkdown', 'source-a'); + + const stats1 = await resolveSymbolEdgesIncremental(engine, { sourceId: 'source-a' }); + expect(stats1.chunks_walked).toBeGreaterThanOrEqual(2); + + const watermarkAfter = await engine.executeRaw<{ count: number }>( + `SELECT COUNT(*)::int AS count FROM content_chunks + WHERE edges_backfilled_at IS NOT NULL`, + [], + ); + expect(watermarkAfter[0]!.count).toBeGreaterThanOrEqual(2); + + // Second run: every chunk already has edges_backfilled_at >= EDGE_EXTRACTOR_VERSION_TS. + const stats2 = await resolveSymbolEdgesIncremental(engine, { sourceId: 'source-a' }); + expect(stats2.chunks_walked).toBe(0); + expect(stats2.edges_examined).toBe(0); + }); +}); + +describe('v0.34 W0c symbol-resolver — source isolation', () => { + test("does not resolve via candidates in a different source", async () => { + await registerSource(engine, 'source-a'); + await registerSource(engine, 'source-b'); + + const pageA = await insertCodePage(engine, 'source-a', 'src/foo.ts'); + const pageB = await insertCodePage(engine, 'source-b', 'src/foo.ts'); + const callerInA = await insertChunk(engine, pageA, 0, 'callerInA', 'function'); + // source-b has the same-named symbol at the same relative file path + await insertChunk(engine, pageB, 0, 'parseMarkdown', 'function'); + // source-a does NOT have a parseMarkdown definition + await insertUnresolvedEdge(engine, callerInA, 'callerInA', 'parseMarkdown', 'source-a'); + + const stats = await resolveSymbolEdgesIncremental(engine, { sourceId: 'source-a' }); + + // The edge stays unresolved — the only same-symbol candidate is in + // source-b, which the resolver must NOT cross to. + expect(stats.edges_unmatched).toBe(1); + expect(stats.edges_resolved).toBe(0); + expect(stats.edges_ambiguous).toBe(0); + }); +}); + +// ───────────────────────────────────────────────────────────────── +// Seeding helpers +// ───────────────────────────────────────────────────────────────── + +async function registerSource(engine: PGLiteEngine, id: string): Promise { + await engine.executeRaw( + `INSERT INTO sources (id, name, local_path, config, created_at) + VALUES ($1, $1, $2, '{}'::jsonb, NOW()) + ON CONFLICT (id) DO NOTHING`, + [id, `/fake/${id}`], + ); +} + +async function insertCodePage(engine: PGLiteEngine, sourceId: string, slug: string): Promise { + const rows = await engine.executeRaw<{ id: number }>( + `INSERT INTO pages (slug, source_id, title, type, page_kind, compiled_truth, frontmatter, updated_at, created_at) + VALUES ($1, $2, $3, 'code', 'code', '', '{}'::jsonb, NOW(), NOW()) + RETURNING id`, + [slug, sourceId, slug], + ); + return rows[0]!.id; +} + +async function insertChunk( + engine: PGLiteEngine, + pageId: number, + chunkIndex: number, + symbolName: string, + symbolType: string, +): Promise { + const rows = await engine.executeRaw<{ id: number }>( + `INSERT INTO content_chunks (page_id, chunk_index, chunk_text, chunk_source, language, symbol_name_qualified, symbol_type) + VALUES ($1, $2, $3, 'compiled_truth', 'typescript', $4, $5) + RETURNING id`, + [pageId, chunkIndex, `// ${symbolName} body`, symbolName, symbolType], + ); + return rows[0]!.id; +} + +async function insertUnresolvedEdge( + engine: PGLiteEngine, + fromChunkId: number, + fromSymbol: string, + toSymbol: string, + sourceId: string, +): Promise { + await engine.executeRaw( + `INSERT INTO code_edges_symbol (from_chunk_id, from_symbol_qualified, to_symbol_qualified, edge_type, source_id, edge_metadata) + VALUES ($1, $2, $3, 'calls', $4, '{}'::jsonb)`, + [fromChunkId, fromSymbol, toSymbol, sourceId], + ); +}