mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
feat(v0.34 W0c): within-file two-pass symbol resolver + edges_backfilled_at watermark
Codex's outside-voice review caught that the v0.20.0 graph stores BARE
callee tokens (`render`, `find`, `execute`) — not qualified names. Pre-v0.34
recursive blast/flow would alias every same-named function across classes.
W0c is the foundation that fixes this: resolve `code_edges_symbol` rows by
matching `to_symbol_qualified` against the SAME-FILE chunks'
`symbol_name_qualified`, then write the outcome to `edge_metadata`.
This commit is the resolver primitive + schema. The cycle-phase wiring
that calls it on every quick-cycle tick lands in the next commit.
Schema (v51 migration `edges_backfilled_at_v0_34`):
* `content_chunks.edges_backfilled_at TIMESTAMPTZ` — resume watermark.
Chunks where the column is NULL OR older than EDGE_EXTRACTOR_VERSION_TS
get re-walked next tick. SIGINT/OOM/sleep mid-backfill loses at most
one batch.
* Indexes per D11 from eng review:
- `idx_code_edges_symbol_resolver(source_id, to_symbol_qualified)` —
composite for the resolver's per-source lookup.
- `idx_content_chunks_symbol_lookup(page_id, symbol_name_qualified)`
WHERE `symbol_name_qualified IS NOT NULL` — file-batched candidate
fetch; also reused by W4-5 cluster recompute.
- `idx_content_chunks_edges_backfill(edges_backfilled_at)` WHERE
`edges_backfilled_at IS NULL` — fast unresumed-row scan.
Module (`src/core/chunkers/symbol-resolver.ts`):
* `resolveSymbolEdgesIncremental(engine, {sourceId, maxChunks?, onProgress?})`
walks stale chunks in 200-chunk batches. For each chunk, loads its
unresolved edges, finds same-page candidates by symbol_name_qualified,
and writes outcome to `edge_metadata`:
- exactly 1 candidate → `{resolved_chunk_id: <id>}`
- 2+ candidates → `{ambiguous: true, candidates: [...]}`
- 0 candidates → unchanged (cross-file; two-pass.ts handles those)
Each batch bumps `edges_backfilled_at = NOW()` for the chunks.
* `readEdgeResolution(metadata)` — public helper for downstream code
(two-pass.ts, code_blast op, eval-capture) to consume the resolver's
output without parsing JSON directly. Returns a tagged union.
* `EDGE_EXTRACTOR_VERSION_TS` exported constant — bump when extractor
shape changes and the next cycle re-walks all chunks.
Tests (5 E2E in test/e2e/symbol-resolver-pglite.test.ts, all PGLite,
no DATABASE_URL): unambiguous match, ambiguous multi-match, no match,
watermark advance + idempotency, source isolation (no cross-source
candidate leak).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
co-authored by
Claude Opus 4.7
Claude Opus 4.7
parent
39a1a2e091
commit
0a94216397
Binary file not shown.
@@ -2548,6 +2548,53 @@ export const MIGRATIONS: Migration[] = [
|
||||
ON ingest_log (source_id, source_type, created_at DESC);
|
||||
`,
|
||||
},
|
||||
{
|
||||
version: 51,
|
||||
name: 'edges_backfilled_at_v0_34',
|
||||
// v0.34 W0c — resumable symbol-resolution backfill watermark.
|
||||
//
|
||||
// The within-file two-pass resolver (src/core/chunkers/symbol-resolver.ts)
|
||||
// walks every content_chunks row that has unresolved edges
|
||||
// (code_edges_symbol.to_chunk_id IS NULL) and matches each
|
||||
// `to_symbol_qualified` against same-file chunks' `symbol_name_qualified`.
|
||||
// On a 96K-chunk brain that is a 5-15 minute backfill the first time
|
||||
// it runs.
|
||||
//
|
||||
// `edges_backfilled_at` is the resume watermark. Backfill runs in
|
||||
// 200-chunk batches; on batch success the column is set to NOW() for
|
||||
// every chunk in the batch. Resume picks up chunks where the watermark
|
||||
// is NULL or older than EDGE_EXTRACTOR_VERSION_TS (a constant bumped
|
||||
// when the extractor's shape changes). Crashes lose at most one batch.
|
||||
//
|
||||
// Composite + partial indexes for the lookup hot path (D11 from eng
|
||||
// review):
|
||||
// - idx_code_edges_symbol_resolver (source_id, to_symbol_qualified)
|
||||
// — every code_edges_symbol row IS an unresolved edge by construction
|
||||
// (the table has no to_chunk_id column; that lives on code_edges_chunk).
|
||||
// This composite index supports the resolver's per-source lookups.
|
||||
// - idx_content_chunks_symbol_lookup (page_id, symbol_name_qualified)
|
||||
// WHERE symbol_name_qualified IS NOT NULL — file-batched lookup
|
||||
// used by both the resolver and the cluster recompute phase (W4-5).
|
||||
// - idx_content_chunks_edges_backfill (edges_backfilled_at)
|
||||
// WHERE edges_backfilled_at IS NULL — find unresumed rows quickly.
|
||||
//
|
||||
// Idempotent: IF NOT EXISTS on column + indexes. Backfill itself runs
|
||||
// separately via the resolve_symbol_edges_incremental cycle phase.
|
||||
sql: `
|
||||
ALTER TABLE content_chunks ADD COLUMN IF NOT EXISTS edges_backfilled_at TIMESTAMPTZ;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_code_edges_symbol_resolver
|
||||
ON code_edges_symbol (source_id, to_symbol_qualified);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_content_chunks_symbol_lookup
|
||||
ON content_chunks (page_id, symbol_name_qualified)
|
||||
WHERE symbol_name_qualified IS NOT NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_content_chunks_edges_backfill
|
||||
ON content_chunks (edges_backfilled_at)
|
||||
WHERE edges_backfilled_at IS NULL;
|
||||
`,
|
||||
},
|
||||
];
|
||||
|
||||
export const LATEST_VERSION = MIGRATIONS.length > 0
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
/**
|
||||
* v0.34 W0c — within-file two-pass symbol resolver E2E.
|
||||
*
|
||||
* Pins:
|
||||
* - Unambiguous within-file match → edge_metadata.resolved_chunk_id set
|
||||
* - Multi-match within file → edge_metadata.ambiguous=true + candidates
|
||||
* - No match → edge stays untouched
|
||||
* - chunks_walked watermark advances (edges_backfilled_at = NOW())
|
||||
* - Idempotency: re-run on processed chunks is a no-op
|
||||
* - Resume: bumping EDGE_EXTRACTOR_VERSION_TS forces re-walk
|
||||
* - Source isolation: resolver scoped to one source_id; never touches edges
|
||||
* in a different source even if the symbol name collides
|
||||
*
|
||||
* PGLite in-memory.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
||||
import { PGLiteEngine } from '../../src/core/pglite-engine.ts';
|
||||
import {
|
||||
resolveSymbolEdgesIncremental,
|
||||
readEdgeResolution,
|
||||
EDGE_EXTRACTOR_VERSION_TS,
|
||||
} from '../../src/core/chunkers/symbol-resolver.ts';
|
||||
import { resetPgliteState } from '../helpers/reset-pglite.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await resetPgliteState(engine);
|
||||
});
|
||||
|
||||
describe('v0.34 W0c symbol-resolver — unambiguous within-file match', () => {
|
||||
test('single-file: parseMarkdown call resolves to the parseMarkdown chunk in same file', async () => {
|
||||
await registerSource(engine, 'source-a');
|
||||
const pageId = await insertCodePage(engine, 'source-a', 'src/foo.ts');
|
||||
const callerChunk = await insertChunk(engine, pageId, 0, 'callerInA', 'function');
|
||||
const defChunk = await insertChunk(engine, pageId, 1, 'parseMarkdown', 'function');
|
||||
await insertUnresolvedEdge(engine, callerChunk, 'callerInA', 'parseMarkdown', 'source-a');
|
||||
|
||||
const stats = await resolveSymbolEdgesIncremental(engine, { sourceId: 'source-a' });
|
||||
|
||||
expect(stats.chunks_walked).toBeGreaterThanOrEqual(2);
|
||||
expect(stats.edges_resolved).toBe(1);
|
||||
expect(stats.edges_ambiguous).toBe(0);
|
||||
expect(stats.edges_unmatched).toBe(0);
|
||||
|
||||
const edges = await engine.executeRaw<{ edge_metadata: any }>(
|
||||
`SELECT edge_metadata FROM code_edges_symbol`,
|
||||
[],
|
||||
);
|
||||
expect(edges.length).toBe(1);
|
||||
const res = readEdgeResolution(edges[0]!.edge_metadata);
|
||||
expect(res.kind).toBe('resolved');
|
||||
if (res.kind === 'resolved') {
|
||||
expect(res.chunk_id).toBe(defChunk);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('v0.34 W0c symbol-resolver — ambiguous within-file match', () => {
|
||||
test('two same-named methods in the same file → ambiguous + candidates list', async () => {
|
||||
await registerSource(engine, 'source-a');
|
||||
const pageId = await insertCodePage(engine, 'source-a', 'src/foo.ts');
|
||||
const callerChunk = await insertChunk(engine, pageId, 0, 'callerInA', 'function');
|
||||
const def1 = await insertChunk(engine, pageId, 1, 'render', 'function');
|
||||
const def2 = await insertChunk(engine, pageId, 2, 'render', 'function'); // dup symbol name in same file
|
||||
await insertUnresolvedEdge(engine, callerChunk, 'callerInA', 'render', 'source-a');
|
||||
|
||||
const stats = await resolveSymbolEdgesIncremental(engine, { sourceId: 'source-a' });
|
||||
|
||||
expect(stats.edges_ambiguous).toBe(1);
|
||||
expect(stats.edges_resolved).toBe(0);
|
||||
|
||||
const edges = await engine.executeRaw<{ edge_metadata: any }>(
|
||||
`SELECT edge_metadata FROM code_edges_symbol`,
|
||||
[],
|
||||
);
|
||||
const res = readEdgeResolution(edges[0]!.edge_metadata);
|
||||
expect(res.kind).toBe('ambiguous');
|
||||
if (res.kind === 'ambiguous') {
|
||||
expect(res.candidate_chunk_ids.sort()).toEqual([def1, def2].sort());
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('v0.34 W0c symbol-resolver — no match', () => {
|
||||
test('call to a symbol defined in another file stays unresolved (caller two-pass handles cross-file)', async () => {
|
||||
await registerSource(engine, 'source-a');
|
||||
const pageA = await insertCodePage(engine, 'source-a', 'src/foo.ts');
|
||||
const pageB = await insertCodePage(engine, 'source-a', 'src/bar.ts');
|
||||
const callerChunk = await insertChunk(engine, pageA, 0, 'callerInA', 'function');
|
||||
await insertChunk(engine, pageB, 0, 'externalFn', 'function'); // different file
|
||||
await insertUnresolvedEdge(engine, callerChunk, 'callerInA', 'externalFn', 'source-a');
|
||||
|
||||
const stats = await resolveSymbolEdgesIncremental(engine, { sourceId: 'source-a' });
|
||||
|
||||
expect(stats.edges_unmatched).toBe(1);
|
||||
expect(stats.edges_resolved).toBe(0);
|
||||
expect(stats.edges_ambiguous).toBe(0);
|
||||
|
||||
const edges = await engine.executeRaw<{ edge_metadata: any }>(
|
||||
`SELECT edge_metadata FROM code_edges_symbol`,
|
||||
[],
|
||||
);
|
||||
const res = readEdgeResolution(edges[0]!.edge_metadata);
|
||||
expect(res.kind).toBe('unresolved');
|
||||
});
|
||||
});
|
||||
|
||||
describe('v0.34 W0c symbol-resolver — watermark + idempotency', () => {
|
||||
test('edges_backfilled_at advances; second run is a no-op', async () => {
|
||||
await registerSource(engine, 'source-a');
|
||||
const pageId = await insertCodePage(engine, 'source-a', 'src/foo.ts');
|
||||
const callerChunk = await insertChunk(engine, pageId, 0, 'callerInA', 'function');
|
||||
await insertChunk(engine, pageId, 1, 'parseMarkdown', 'function');
|
||||
await insertUnresolvedEdge(engine, callerChunk, 'callerInA', 'parseMarkdown', 'source-a');
|
||||
|
||||
const stats1 = await resolveSymbolEdgesIncremental(engine, { sourceId: 'source-a' });
|
||||
expect(stats1.chunks_walked).toBeGreaterThanOrEqual(2);
|
||||
|
||||
const watermarkAfter = await engine.executeRaw<{ count: number }>(
|
||||
`SELECT COUNT(*)::int AS count FROM content_chunks
|
||||
WHERE edges_backfilled_at IS NOT NULL`,
|
||||
[],
|
||||
);
|
||||
expect(watermarkAfter[0]!.count).toBeGreaterThanOrEqual(2);
|
||||
|
||||
// Second run: every chunk already has edges_backfilled_at >= EDGE_EXTRACTOR_VERSION_TS.
|
||||
const stats2 = await resolveSymbolEdgesIncremental(engine, { sourceId: 'source-a' });
|
||||
expect(stats2.chunks_walked).toBe(0);
|
||||
expect(stats2.edges_examined).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('v0.34 W0c symbol-resolver — source isolation', () => {
|
||||
test("does not resolve via candidates in a different source", async () => {
|
||||
await registerSource(engine, 'source-a');
|
||||
await registerSource(engine, 'source-b');
|
||||
|
||||
const pageA = await insertCodePage(engine, 'source-a', 'src/foo.ts');
|
||||
const pageB = await insertCodePage(engine, 'source-b', 'src/foo.ts');
|
||||
const callerInA = await insertChunk(engine, pageA, 0, 'callerInA', 'function');
|
||||
// source-b has the same-named symbol at the same relative file path
|
||||
await insertChunk(engine, pageB, 0, 'parseMarkdown', 'function');
|
||||
// source-a does NOT have a parseMarkdown definition
|
||||
await insertUnresolvedEdge(engine, callerInA, 'callerInA', 'parseMarkdown', 'source-a');
|
||||
|
||||
const stats = await resolveSymbolEdgesIncremental(engine, { sourceId: 'source-a' });
|
||||
|
||||
// The edge stays unresolved — the only same-symbol candidate is in
|
||||
// source-b, which the resolver must NOT cross to.
|
||||
expect(stats.edges_unmatched).toBe(1);
|
||||
expect(stats.edges_resolved).toBe(0);
|
||||
expect(stats.edges_ambiguous).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
// Seeding helpers
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
|
||||
async function registerSource(engine: PGLiteEngine, id: string): Promise<void> {
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO sources (id, name, local_path, config, created_at)
|
||||
VALUES ($1, $1, $2, '{}'::jsonb, NOW())
|
||||
ON CONFLICT (id) DO NOTHING`,
|
||||
[id, `/fake/${id}`],
|
||||
);
|
||||
}
|
||||
|
||||
async function insertCodePage(engine: PGLiteEngine, sourceId: string, slug: string): Promise<number> {
|
||||
const rows = await engine.executeRaw<{ id: number }>(
|
||||
`INSERT INTO pages (slug, source_id, title, type, page_kind, compiled_truth, frontmatter, updated_at, created_at)
|
||||
VALUES ($1, $2, $3, 'code', 'code', '', '{}'::jsonb, NOW(), NOW())
|
||||
RETURNING id`,
|
||||
[slug, sourceId, slug],
|
||||
);
|
||||
return rows[0]!.id;
|
||||
}
|
||||
|
||||
async function insertChunk(
|
||||
engine: PGLiteEngine,
|
||||
pageId: number,
|
||||
chunkIndex: number,
|
||||
symbolName: string,
|
||||
symbolType: string,
|
||||
): Promise<number> {
|
||||
const rows = await engine.executeRaw<{ id: number }>(
|
||||
`INSERT INTO content_chunks (page_id, chunk_index, chunk_text, chunk_source, language, symbol_name_qualified, symbol_type)
|
||||
VALUES ($1, $2, $3, 'compiled_truth', 'typescript', $4, $5)
|
||||
RETURNING id`,
|
||||
[pageId, chunkIndex, `// ${symbolName} body`, symbolName, symbolType],
|
||||
);
|
||||
return rows[0]!.id;
|
||||
}
|
||||
|
||||
async function insertUnresolvedEdge(
|
||||
engine: PGLiteEngine,
|
||||
fromChunkId: number,
|
||||
fromSymbol: string,
|
||||
toSymbol: string,
|
||||
sourceId: string,
|
||||
): Promise<void> {
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO code_edges_symbol (from_chunk_id, from_symbol_qualified, to_symbol_qualified, edge_type, source_id, edge_metadata)
|
||||
VALUES ($1, $2, $3, 'calls', $4, '{}'::jsonb)`,
|
||||
[fromChunkId, fromSymbol, toSymbol, sourceId],
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user