mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
* v0.40.4.0 feat(sync): parallel sync --all + per-source lock invariant + sources status dashboard (productionized from PR #1314) Lands the community-authored PR #1314 with the structural fixes Codex's outside-voice review caught: the original PR's lock-id change only fired inside the --all parallel path, which would have introduced a worse race than the global-lock contention it fixed (sync --all on per-source lock racing against sync --source foo on the still-global lock). The landed version makes the per-source lock the invariant for every source-scoped sync, paired with withRefreshingLock for sources that exceed 30 minutes. What's new - gbrain sync --all parallel fan-out via continuous worker pool (D2); --parallel N flag, default min(sourceCount, --workers, 4); per-source [<source-id>] line prefix via AsyncLocalStorage (D6 + D12 + D13); stable --json envelope {schema_version:1, ...} on stdout with banners on stderr (D4 + D14); --skip-failed/--retry-failed reject under --parallel > 1 (D15 — sync-failures.jsonl is brain-global today; source-scoping filed as v0.40.4 TODO). - gbrain sources status [--json] read-only dashboard (D3 — sibling to sources list/add/remove/archive, not a sync flag, so reads + writes don't share a verb). Counts pages + chunks + embedding coverage per source. Active embedding column resolved via the registry (D16) so Voyage / multimodal brains see the right column. Archived sources excluded by caller filter. - Connection-budget stderr warning when parallel × workers × 2 > 16 with the formula in the message text (D1 + D10 — Codex P0 #3: each per-file worker opens its own PostgresEngine with poolSize=2, so the multiplication factor is 2, not 1). The load-bearing structural fix - performSync defaults to per-source lock id (gbrain-sync:<sourceId>) whenever opts.sourceId is set + wraps in withRefreshingLock. Legacy single-default-source brains keep the bare tryAcquireDbLock(SYNC_LOCK_ID) path for back-compat. - Dashboard SQL is the canonical content_chunks ch JOIN pages pg ON pg.id = ch.page_id WHERE pg.deleted_at IS NULL shape — the original PR shipped chunks ch JOIN ON page_slug, which would have crashed on PGLite parse and silently zeroed on Postgres via a swallow-catch. Errors from the dashboard SQL propagate (no silent zero-counts on real DB errors). Tests - New test/console-prefix.test.ts — 8 cases pinning ALS propagation, nested wraps, embedded-newline prefixing, back-compat fast path. - New test/sync-all-parallel.test.ts (replaces PR's stubbed tests) — 16 cases covering resolveParallelism, per-source lock format, buildSyncStatusReport SQL math + error propagation + envelope shape, connection-budget math, per-source prefix routing. - New test/e2e/sync-status-pglite.test.ts — IRON RULE regression: real PGLite seeds 2 sources × pages × chunks (mixed embedded/unembedded, 1 soft-deleted, 1 archived source). Validates SQL excludes both AND the active embedding column is the one used. This is the case that would have caught the PR's original broken SQL. Compatibility - No schema changes. No new dependencies. - Single-source / non-`--all` paths: bit-for-bit identical to v0.40.2. - PGLite users get serial behavior (single-connection engine). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> Co-Authored-By: garrytan-agents <garrytan-agents@users.noreply.github.com> * v0.40.6.0 — version bump for ship (skipping 0.40.4 + 0.40.5 for in-flight work) Reserves v0.40.4 + v0.40.5 slots for parallel waves (salem's graph-signals work and any other in-flight branches) and lands this PR's parallel-sync work at v0.40.6.0. No code change beyond the version triple and the TODOS / CLAUDE.md / CHANGELOG cross-references which were updated from "v0.40.4" to "v0.41+" to match the new follow-up version. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com> Co-authored-by: garrytan-agents <garrytan-agents@users.noreply.github.com>
154 lines
6.3 KiB
TypeScript
154 lines
6.3 KiB
TypeScript
/**
|
|
* Tests for the v0.40.3.0 per-source console-prefix helper.
|
|
*
|
|
* Why these exist:
|
|
* AsyncLocalStorage propagation through await boundaries is what makes
|
|
* `withSourcePrefix(src.id, () => performSync(...))` correct without
|
|
* manual threading. If the propagation breaks (e.g. via setImmediate or
|
|
* a Promise resolver that bypasses ALS), every `slog` inside performSync
|
|
* loses its prefix and operators see interleaved unreadable output under
|
|
* `--parallel > 1`. These cases pin the contract.
|
|
*
|
|
* The line-splitting math (embedded newlines each get their own prefix)
|
|
* is the difference between greppable output and a wall of text where
|
|
* only the first line of every multi-line emitter has a prefix.
|
|
*/
|
|
|
|
import { describe, expect, test } from 'bun:test';
|
|
import {
|
|
getSourcePrefix,
|
|
slog,
|
|
serr,
|
|
withSourcePrefix,
|
|
} from '../src/core/console-prefix.ts';
|
|
|
|
// Capture stdout/stderr writes for the duration of a callback. Restores
|
|
// the original write fns even on throw. Returns the captured chunks.
|
|
function captureStdio<T>(fn: () => T | Promise<T>): Promise<{ stdout: string; stderr: string; result: T }> {
|
|
const stdoutChunks: string[] = [];
|
|
const stderrChunks: string[] = [];
|
|
const origStdout = process.stdout.write.bind(process.stdout);
|
|
const origStderr = process.stderr.write.bind(process.stderr);
|
|
process.stdout.write = ((chunk: string | Uint8Array): boolean => {
|
|
stdoutChunks.push(typeof chunk === 'string' ? chunk : Buffer.from(chunk).toString('utf-8'));
|
|
return true;
|
|
}) as typeof process.stdout.write;
|
|
process.stderr.write = ((chunk: string | Uint8Array): boolean => {
|
|
stderrChunks.push(typeof chunk === 'string' ? chunk : Buffer.from(chunk).toString('utf-8'));
|
|
return true;
|
|
}) as typeof process.stderr.write;
|
|
return Promise.resolve(fn())
|
|
.then((result) => ({ stdout: stdoutChunks.join(''), stderr: stderrChunks.join(''), result }))
|
|
.finally(() => {
|
|
process.stdout.write = origStdout;
|
|
process.stderr.write = origStderr;
|
|
});
|
|
}
|
|
|
|
describe('withSourcePrefix / getSourcePrefix', () => {
|
|
test('prefix is applied inside the wrap', async () => {
|
|
let observed: string | null = 'unset';
|
|
await withSourcePrefix('media-corpus', async () => {
|
|
observed = getSourcePrefix();
|
|
});
|
|
expect(observed).toBe('media-corpus');
|
|
});
|
|
|
|
test('no prefix outside the wrap', async () => {
|
|
// Sanity: getSourcePrefix returns null when called outside any wrap.
|
|
// This is what guarantees slog/serr fall through to bare console.log
|
|
// for single-source / non-parallel callers (back-compat invariant).
|
|
expect(getSourcePrefix()).toBeNull();
|
|
});
|
|
|
|
test('nested wrap uses innermost prefix; outer restored on exit', async () => {
|
|
const observed: { outerBefore: string | null; inner: string | null; outerAfter: string | null } = {
|
|
outerBefore: null,
|
|
inner: null,
|
|
outerAfter: null,
|
|
};
|
|
await withSourcePrefix('outer', async () => {
|
|
observed.outerBefore = getSourcePrefix();
|
|
await withSourcePrefix('inner', async () => {
|
|
observed.inner = getSourcePrefix();
|
|
});
|
|
observed.outerAfter = getSourcePrefix();
|
|
});
|
|
expect(observed.outerBefore).toBe('outer');
|
|
expect(observed.inner).toBe('inner');
|
|
expect(observed.outerAfter).toBe('outer');
|
|
});
|
|
|
|
test('prefix propagates through await boundaries (the load-bearing ALS contract)', async () => {
|
|
// This is the case that justifies AsyncLocalStorage over a global
|
|
// variable. If propagation breaks, every async function called from
|
|
// inside performSync loses its prefix and the whole feature is
|
|
// theatrical. We assert by awaiting through Promise.resolve and a
|
|
// setImmediate microtask — both common patterns inside the sync path.
|
|
const observed: Array<string | null> = [];
|
|
await withSourcePrefix('foo', async () => {
|
|
observed.push(getSourcePrefix());
|
|
await Promise.resolve();
|
|
observed.push(getSourcePrefix());
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
observed.push(getSourcePrefix());
|
|
});
|
|
expect(observed).toEqual(['foo', 'foo', 'foo']);
|
|
});
|
|
});
|
|
|
|
describe('slog / serr line prefixing', () => {
|
|
test('slog under a wrap prefixes a single-line string and writes to stdout', async () => {
|
|
const { stdout, stderr } = await captureStdio(async () => {
|
|
await withSourcePrefix('media-corpus', async () => {
|
|
slog('phase started');
|
|
});
|
|
});
|
|
expect(stdout).toBe('[media-corpus] phase started\n');
|
|
expect(stderr).toBe('');
|
|
});
|
|
|
|
test('serr under a wrap prefixes and writes to stderr', async () => {
|
|
const { stdout, stderr } = await captureStdio(async () => {
|
|
await withSourcePrefix('zion-brain', async () => {
|
|
serr('warn: skipping');
|
|
});
|
|
});
|
|
expect(stdout).toBe('');
|
|
expect(stderr).toBe('[zion-brain] warn: skipping\n');
|
|
});
|
|
|
|
test('embedded newlines each get their own prefix (greppable multi-line output)', async () => {
|
|
// Without per-line prefixing, only the first line of a multi-line
|
|
// emit would carry the source tag; the rest would be ambiguous under
|
|
// interleaved parallel output. This case pins the kubectl-style
|
|
// semantics that make `grep '[media-corpus]'` actually work.
|
|
const { stdout } = await captureStdio(async () => {
|
|
await withSourcePrefix('media-corpus', async () => {
|
|
slog('phase started\n details: x\n details: y');
|
|
});
|
|
});
|
|
expect(stdout).toBe(
|
|
'[media-corpus] phase started\n[media-corpus] details: x\n[media-corpus] details: y\n',
|
|
);
|
|
});
|
|
|
|
test('slog outside a wrap is identical to console.log (back-compat invariant)', async () => {
|
|
// Single-source / non-parallel callers (single-source gbrain sync,
|
|
// doctor, every existing caller that doesn't opt into withSourcePrefix)
|
|
// must see bit-for-bit identical output. The fast path through
|
|
// console.log preserves that — no prefix, no extra formatting.
|
|
const origLog = console.log;
|
|
const captured: unknown[][] = [];
|
|
// eslint-disable-next-line no-console
|
|
console.log = (...args: unknown[]) => { captured.push(args); };
|
|
try {
|
|
slog('plain message', { foo: 1 });
|
|
expect(captured).toEqual([['plain message', { foo: 1 }]]);
|
|
} finally {
|
|
// eslint-disable-next-line no-console
|
|
console.log = origLog;
|
|
}
|
|
});
|
|
});
|