/** * E2E test for parallel sync against real Postgres. * * T2 — happy path: 60-file sync at concurrency=4 against PostgresEngine * actually constructs N worker engines, imports correctly, and does * not leak connections (probe pg_stat_activity before/after). * P4 — benchmark: serial vs concurrency=4 timing on the same fixture so * the v0.22.13 CHANGELOG can quote a real number instead of "~4×". * * Gated on DATABASE_URL. Run via: * docker run -d --name gbrain-test-pg -e POSTGRES_USER=postgres \ * -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=gbrain_test \ * -p 5435:5432 pgvector/pgvector:pg16 * DATABASE_URL=postgresql://postgres:postgres@localhost:5435/gbrain_test \ * bun test test/e2e/sync-parallel.test.ts */ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from 'fs'; import { join } from 'path'; import { tmpdir } from 'os'; import { execSync } from 'child_process'; import { hasDatabase, setupDB, teardownDB, getEngine, getConn } from './helpers.ts'; const skip = !hasDatabase(); const describeE2E = skip ? describe.skip : describe; if (skip) { console.log('Skipping E2E sync-parallel tests (DATABASE_URL not set)'); } function seedRepo(repoPath: string, fileCount: number): string { execSync('git init', { cwd: repoPath, stdio: 'pipe' }); execSync('git config user.email "test@test.com"', { cwd: repoPath, stdio: 'pipe' }); execSync('git config user.name "Test"', { cwd: repoPath, stdio: 'pipe' }); mkdirSync(join(repoPath, 'people'), { recursive: true }); for (let i = 0; i < fileCount; i++) { writeFileSync(join(repoPath, `people/p${i}.md`), [ '---', 'type: person', `title: Person ${i}`, '---', '', `Person ${i} body — some text long enough to chunk.`, `Iteration index ${i}, generated by sync-parallel E2E.`, ].join('\n')); } execSync('git add -A && git commit -m "initial"', { cwd: repoPath, stdio: 'pipe' }); return execSync('git rev-parse HEAD', { cwd: repoPath, encoding: 'utf-8' }).trim(); } async function activeConnections(): Promise { const conn = getConn(); const rows = await conn.unsafe(` SELECT count(*) AS n FROM pg_stat_activity WHERE datname = current_database() AND state IS NOT NULL `) as Array<{ n: string }>; return parseInt(rows[0]?.n ?? '0', 10); } describeE2E('E2E sync-parallel: T2 happy path + leak probe', () => { let repoPath: string; beforeAll(async () => { await setupDB(); }, 30_000); afterAll(async () => { if (repoPath) rmSync(repoPath, { recursive: true, force: true }); await teardownDB(); }); test('60-file Postgres sync at concurrency=4 imports all + no connection leak', async () => { repoPath = mkdtempSync(join(tmpdir(), 'gbrain-e2e-par-')); seedRepo(repoPath, 60); const before = await activeConnections(); const { performSync } = await import('../../src/commands/sync.ts'); const engine = getEngine(); const result = await performSync(engine, { repoPath, noPull: true, noEmbed: true, concurrency: 4, }); // First sync routes through performFullSync (delegates to runImport which // also accepts --workers); status is 'first_sync'. expect(result.status).toBe('first_sync'); const after = await activeConnections(); // Allow some slack — the helper engine + sync's normal pool stay open. // Worker engines (4 × 2 = 8 connections) MUST have closed; if they // hadn't, after - before would be at least 8. expect(after - before).toBeLessThan(4); // Verify pages are actually in the DB (via raw SQL — engine API also works). const conn = getConn(); const pageRows = await conn.unsafe( `SELECT count(*) AS n FROM pages WHERE slug LIKE 'people/p%'`, ) as Array<{ n: string }>; const count = parseInt(pageRows[0]?.n ?? '0', 10); expect(count).toBe(60); }, 60_000); }); describeE2E('E2E sync-parallel: P4 benchmark serial vs concurrency=4', () => { let repoSerial: string; let repoParallel: string; beforeAll(async () => { await setupDB(); }, 30_000); afterAll(async () => { if (repoSerial) rmSync(repoSerial, { recursive: true, force: true }); if (repoParallel) rmSync(repoParallel, { recursive: true, force: true }); await teardownDB(); }); test('120-file benchmark: report serial and parallel wall-clock', async () => { // Two separate repos so neither sync's chunks bleed into the other. repoSerial = mkdtempSync(join(tmpdir(), 'gbrain-bench-serial-')); repoParallel = mkdtempSync(join(tmpdir(), 'gbrain-bench-parallel-')); seedRepo(repoSerial, 120); seedRepo(repoParallel, 120); const { performSync } = await import('../../src/commands/sync.ts'); const engine = getEngine(); // Truncate between runs to keep the benchmark honest. const conn = getConn(); const t1 = Date.now(); await performSync(engine, { repoPath: repoSerial, noPull: true, noEmbed: true, concurrency: 1, }); const serialMs = Date.now() - t1; // Wipe pages before second run so neither one is "incremental". await conn.unsafe(`TRUNCATE pages CASCADE`); await conn.unsafe(`TRUNCATE config CASCADE`); const t2 = Date.now(); await performSync(engine, { repoPath: repoParallel, noPull: true, noEmbed: true, concurrency: 4, }); const parallelMs = Date.now() - t2; const speedup = (serialMs / parallelMs).toFixed(2); // Emit as a single line stdout consumers can grep for. console.log(`SYNC_PARALLEL_BENCH 120 files | serial=${serialMs}ms | parallel(4)=${parallelMs}ms | speedup=${speedup}x`); // Soft assertion: parallel must not be slower than serial. The actual // speedup ratio depends heavily on Postgres latency profile and is what // the CHANGELOG quotes — don't gate the test on a specific multiplier. expect(parallelMs).toBeLessThanOrEqual(serialMs * 1.5); // +50% slack for noisy CI }, 120_000); }); describeE2E('E2E sync-parallel: T18 --timeout returns partial; last_commit unchanged', () => { let repoPath: string; beforeAll(async () => { await setupDB(); }, 30_000); afterAll(async () => { if (repoPath) rmSync(repoPath, { recursive: true, force: true }); await teardownDB(); }); test('signal aborted mid-import returns partial and does not advance last_commit', async () => { // v0.41.13.0 (T18 / D-V4-mech-10): real-Postgres E2E for the // --timeout partial-status contract. PGLite tests cover the // single-source AbortSignal threading in test/sync-break-lock-all.test.ts; // this case verifies the same contract on the actual Postgres engine // because parallelEligible excludes PGLite from the worker fan-out // and the bookmark-write semantic uses real Postgres timestamp + index // behavior. repoPath = mkdtempSync(join(tmpdir(), 'gbrain-e2e-timeout-')); seedRepo(repoPath, 200); const { performSync } = await import('../../src/commands/sync.ts'); const engine = getEngine(); // Register a source so per-source last_commit lives in `sources`. const conn = getConn(); await conn.unsafe( `INSERT INTO sources (id, name, local_path) VALUES ($1, $2, $3) ON CONFLICT (id) DO UPDATE SET local_path = EXCLUDED.local_path`, ['e2e-timeout-source', 'e2e-timeout-source', repoPath], ); // Fire abort immediately. With a 200-file diff, performSync's per-file // abort check at the top of the import loop fires before file 1 starts, // so files_imported should be 0 and last_commit should stay null. const controller = new AbortController(); controller.abort(); const result = await performSync(engine, { repoPath, sourceId: 'e2e-timeout-source', noPull: true, noEmbed: true, noExtract: true, concurrency: 1, signal: controller.signal, }); expect(result.status).toBe('partial'); expect(result.reason).toBeDefined(); // last_commit must NOT have advanced (D-V3-1 invariant — partial // fires strictly before the writeSyncAnchor call). const rows = await conn.unsafe( `SELECT last_commit FROM sources WHERE id = $1`, ['e2e-timeout-source'], ) as Array<{ last_commit: string | null }>; expect(rows[0]?.last_commit).toBeNull(); }, 60_000); test('signal aborted after a few imports leaves last_commit unchanged and reports partial files_imported', async () => { // Rebuild a fresh repo for this test; the prior describe path uses // its own repoPath variable. const repo2 = mkdtempSync(join(tmpdir(), 'gbrain-e2e-timeout-partial-')); try { seedRepo(repo2, 50); const { performSync } = await import('../../src/commands/sync.ts'); const engine = getEngine(); const conn = getConn(); await conn.unsafe( `INSERT INTO sources (id, name, local_path) VALUES ($1, $2, $3) ON CONFLICT (id) DO UPDATE SET local_path = EXCLUDED.local_path`, ['e2e-timeout-partial', 'e2e-timeout-partial', repo2], ); // Schedule abort 250ms in. On a 50-file repo with real Postgres // round-trips per import, some files persist before abort fires. // We assert that: // - status is partial OR first_sync (race-tolerant — if Postgres // is fast enough that all 50 imports finish in <250ms, the run // completes successfully which is also a valid outcome) // - if partial: filesImported is bounded between 1 and 49 // - if partial: last_commit is null (never advanced past partial) const controller = new AbortController(); setTimeout(() => controller.abort(), 250).unref(); const result = await performSync(engine, { repoPath: repo2, sourceId: 'e2e-timeout-partial', noPull: true, noEmbed: true, noExtract: true, concurrency: 1, signal: controller.signal, }); if (result.status === 'partial') { expect(result.filesImported).toBeGreaterThanOrEqual(0); expect(result.filesImported).toBeLessThanOrEqual(50); const rows = await conn.unsafe( `SELECT last_commit FROM sources WHERE id = $1`, ['e2e-timeout-partial'], ) as Array<{ last_commit: string | null }>; expect(rows[0]?.last_commit).toBeNull(); } else { // first_sync or synced — sub-250ms full run; not a contract violation. // The point of the test is that IF partial happens, the invariants hold. expect(['first_sync', 'synced']).toContain(result.status); } } finally { rmSync(repo2, { recursive: true, force: true }); } }, 60_000); });