mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
v0.42.20.0 fix: reliability wave — PGLite capture lock-pin + Postgres reconnect race + search embed-hang (#1762 #1745 #1775) (#1810)
* fix(core): drain fire-and-forget sinks before disconnect via a background-work registry (#1762) New src/core/background-work.ts registry (Map<name,drainer>, ordered drain, awaited abort). facts-queue (order 0, abort=shutdown), last-retrieved (1), and eval-capture (3, now self-tracked) register as sinks. Both CLI exit paths (op-dispatch finally + handleCliOnly finally) drain the registry before engine.disconnect() so a PGLite db.close() can't race in-flight work into the re-pump busy-loop that pinned the single-writer lock. Op-dispatch error path converts process.exit(1) to exitCode+return so the finally still drains. * fix(ai): bound every outbound AI call so a stalled provider can't hang (#1762/#1775) withDefaultTimeout composes a per-touchpoint default deadline (chat 300s, embed/multimodal 60s) with any caller signal via AbortSignal.any. Applied at the SDK call layer (chat generateText, expand generateObject, OCR, per-sub-batch embed) — covers native-anthropic + retries — plus per-request multimodal fetch. embedQuery forwards abortSignal. Env: GBRAIN_AI_{CHAT,EMBED,MULTIMODAL}_TIMEOUT_MS. * fix(postgres): module-mode reconnect preserves the shared singleton (#1745) reconnect() branches on connection style. Module-singleton engines re-establish idempotently via db.connect() (no-op when alive) + refresh the ConnectionManager read pool, never db.disconnect() — so a transient blip no longer nulls the shared sql out from under concurrent ops (which threw 'connect() has not been called'). Fail-loud on real connect failure. Instance pools keep teardown+recreate. * fix(search): bound the query-time embed so a stall falls back to keyword (#1775) search/query default to cheap-hybrid (embeds the query); a stalled provider made the embed never settle, so the keyword fallback never engaged and the command force-exited with no output. One shared QueryEmbedDeadline (6s, floored 2s per embed) covers both the cache-lookup and inner embeds via embedQueryBounded (abortSignal + Promise.race) → existing keyword fallback engages. Also registers the search-cache background-work drainer (now bounded). Env: GBRAIN_QUERY_EMBED_TIMEOUT_MS. * test+chore: reliability wave tests + v0.42.11.0 (#1762 #1745 #1775) New: background-work registry unit, query-embed deadline unit, eval-capture drain unit, postgres reconnect E2E (#1745), gbrain capture exit-clean case in the PGLite serial test. Updated fix-wave-structural assertions to the registry shape. VERSION/package.json/CHANGELOG -> 0.42.11.0; TODOS retrofit marked done. Incorporates + hardens PR #1763 (drain-before-disconnect + embed fetch timeout); the residual hung-Haiku hole is closed by the facts shutdown() abort belt. Co-Authored-By: ElliotDrel <noreply@github.com> Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * docs: document background-work registry + v0.42.11.0 reliability wave in CLAUDE.md (regen llms) * chore: bump release version 0.42.11.0 -> 0.42.20.0 Rename the reliability-wave release version per request. Trio (VERSION / package.json / CHANGELOG) reconciled; in-code version-tag comments and test fixtures updated; llms regenerated. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: ElliotDrel <noreply@github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
co-authored by
ElliotDrel
Claude Opus 4.8
ElliotDrel
Claude Opus 4.8
parent
3d2add15d9
commit
ec5fed2921
@@ -0,0 +1,136 @@
|
||||
/**
|
||||
* v0.42.20.0 (#1762) — background-work registry unit tests.
|
||||
*
|
||||
* Pure, no DB / no engine: drives the registry via the __registerDrainerForTest
|
||||
* seam. Pins the contracts the reliability wave depends on:
|
||||
* - drains in explicit (order, name) order — facts (0) first
|
||||
* - a drainer reporting unfinished>0 has its abort() AWAITED
|
||||
* - a throwing drainer doesn't block the others
|
||||
* - empty registry is a fast no-op
|
||||
* - Map idempotency: re-registering the same name REPLACES (no duplicate)
|
||||
* - the unregister handle removes it
|
||||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import {
|
||||
drainAllBackgroundWorkForCliExit,
|
||||
__registerDrainerForTest,
|
||||
__listDrainerNamesForTest,
|
||||
type BackgroundWorkDrainer,
|
||||
} from '../../src/core/background-work.ts';
|
||||
|
||||
function makeRecorder() {
|
||||
const calls: string[] = [];
|
||||
return { calls };
|
||||
}
|
||||
|
||||
describe('background-work registry', () => {
|
||||
test('drains in explicit (order, name) order; facts-order-0 first', async () => {
|
||||
const { calls } = makeRecorder();
|
||||
const mk = (name: string, order: number): [BackgroundWorkDrainer, () => void] => {
|
||||
const d: BackgroundWorkDrainer = {
|
||||
name,
|
||||
order,
|
||||
drain: async () => { calls.push(`drain:${name}`); return { unfinished: 0 }; },
|
||||
};
|
||||
return [d, __registerDrainerForTest(d)];
|
||||
};
|
||||
const [, u1] = mk('zzz', 2);
|
||||
const [, u0] = mk('facts', 0);
|
||||
const [, u15] = mk('mid', 1);
|
||||
try {
|
||||
await drainAllBackgroundWorkForCliExit({ timeoutMs: 50 });
|
||||
// Only assert ordering among the ones we registered (production sinks may
|
||||
// also be registered when their modules were imported).
|
||||
const ours = calls.filter((c) => ['drain:facts', 'drain:mid', 'drain:zzz'].includes(c));
|
||||
expect(ours).toEqual(['drain:facts', 'drain:mid', 'drain:zzz']);
|
||||
} finally {
|
||||
u0(); u15(); u1();
|
||||
}
|
||||
});
|
||||
|
||||
test('abort() is AWAITED only when unfinished>0', async () => {
|
||||
const seq: string[] = [];
|
||||
const withUnfinished: BackgroundWorkDrainer = {
|
||||
name: 'test-straggler',
|
||||
order: 0,
|
||||
drain: async () => { seq.push('drain'); return { unfinished: 3 }; },
|
||||
abort: async () => {
|
||||
await new Promise((r) => setTimeout(r, 5));
|
||||
seq.push('abort-done');
|
||||
},
|
||||
};
|
||||
const noUnfinished: BackgroundWorkDrainer = {
|
||||
name: 'test-clean',
|
||||
order: 1,
|
||||
drain: async () => { seq.push('drain-clean'); return { unfinished: 0 }; },
|
||||
abort: async () => { seq.push('abort-clean-SHOULD-NOT-RUN'); },
|
||||
};
|
||||
const u1 = __registerDrainerForTest(withUnfinished);
|
||||
const u2 = __registerDrainerForTest(noUnfinished);
|
||||
try {
|
||||
await drainAllBackgroundWorkForCliExit({ timeoutMs: 50 });
|
||||
// straggler: drain then abort (awaited → abort-done present); clean: no abort.
|
||||
expect(seq).toContain('drain');
|
||||
expect(seq).toContain('abort-done');
|
||||
expect(seq).toContain('drain-clean');
|
||||
expect(seq).not.toContain('abort-clean-SHOULD-NOT-RUN');
|
||||
// abort-done comes after its own drain (awaited).
|
||||
expect(seq.indexOf('abort-done')).toBeGreaterThan(seq.indexOf('drain'));
|
||||
} finally {
|
||||
u1(); u2();
|
||||
}
|
||||
});
|
||||
|
||||
test('a throwing drainer does not block the others', async () => {
|
||||
const seen: string[] = [];
|
||||
const boom: BackgroundWorkDrainer = {
|
||||
name: 'test-boom',
|
||||
order: 0,
|
||||
drain: async () => { throw new Error('drain blew up'); },
|
||||
};
|
||||
const ok: BackgroundWorkDrainer = {
|
||||
name: 'test-ok-after-boom',
|
||||
order: 1,
|
||||
drain: async () => { seen.push('ok'); return { unfinished: 0 }; },
|
||||
};
|
||||
const u1 = __registerDrainerForTest(boom);
|
||||
const u2 = __registerDrainerForTest(ok);
|
||||
try {
|
||||
// Must not reject despite the throwing drainer.
|
||||
await drainAllBackgroundWorkForCliExit({ timeoutMs: 50 });
|
||||
expect(seen).toContain('ok');
|
||||
} finally {
|
||||
u1(); u2();
|
||||
}
|
||||
});
|
||||
|
||||
test('Map idempotency: re-registering the same name replaces, no duplicate', () => {
|
||||
const before = __listDrainerNamesForTest().filter((n) => n === 'test-dup').length;
|
||||
expect(before).toBe(0);
|
||||
const d1: BackgroundWorkDrainer = { name: 'test-dup', order: 0, drain: async () => ({ unfinished: 0 }) };
|
||||
const d2: BackgroundWorkDrainer = { name: 'test-dup', order: 9, drain: async () => ({ unfinished: 0 }) };
|
||||
const u1 = __registerDrainerForTest(d1);
|
||||
const u2 = __registerDrainerForTest(d2);
|
||||
try {
|
||||
const count = __listDrainerNamesForTest().filter((n) => n === 'test-dup').length;
|
||||
expect(count).toBe(1); // replaced, not duplicated
|
||||
} finally {
|
||||
u1(); u2();
|
||||
}
|
||||
});
|
||||
|
||||
test('unregister handle removes the drainer', () => {
|
||||
const d: BackgroundWorkDrainer = { name: 'test-unreg', order: 0, drain: async () => ({ unfinished: 0 }) };
|
||||
const unreg = __registerDrainerForTest(d);
|
||||
expect(__listDrainerNamesForTest()).toContain('test-unreg');
|
||||
unreg();
|
||||
expect(__listDrainerNamesForTest()).not.toContain('test-unreg');
|
||||
});
|
||||
|
||||
test('empty registry (no test drainers) resolves fast', async () => {
|
||||
// Production sinks may be registered, but they fast-path on empty pending
|
||||
// sets. This just asserts the call resolves without throwing.
|
||||
await drainAllBackgroundWorkForCliExit({ timeoutMs: 10 });
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -227,6 +227,46 @@ describe('v0.41.8.0 — PGLite CLI read commands exit cleanly (#1247/#1269/#1290
|
||||
}, 30_000);
|
||||
});
|
||||
|
||||
describe('v0.42.20.0 — gbrain capture (CLI_ONLY) exits cleanly + frees the lock (#1762)', () => {
|
||||
test('multi-chunk capture exits 0 within 25s AND a later command runs lock-free', async () => {
|
||||
// The #1762 repro: capture on a multi-chunk page enqueues a fire-and-forget
|
||||
// facts:absorb job, then handleCliOnly's finally disconnects mid-job →
|
||||
// PGLite db.close() busy-loop pins the single-writer lock. The registry
|
||||
// drain-before-disconnect (Fix 0.A + Fix 1) closes it. Without an API key
|
||||
// the facts job is a fast no-op, so this is a wiring regression guard: the
|
||||
// drain+disconnect path runs and capture exits cleanly + the lock is free.
|
||||
const body = Array.from({ length: 12 }, (_, i) =>
|
||||
`## Section ${i}\n\nThis is paragraph ${i} of a deliberately long meeting note ` +
|
||||
`with enough prose to split into multiple chunks. Foxtrot tango whiskey ${i}. ` +
|
||||
`The recursive chunker targets a few hundred tokens per chunk, so a dozen of ` +
|
||||
`these sections guarantees a multi-chunk page for the capture path.\n`,
|
||||
).join('\n');
|
||||
const capFile = join(tmpHome, 'capture-input.md');
|
||||
writeFileSync(capFile, `---\ntitle: Capture Meeting\n---\n${body}\n`, 'utf-8');
|
||||
|
||||
const cap = await runWithTimeout(
|
||||
['capture', '--file', capFile, '--slug', 'meetings/capture-test', '--type', 'meeting', '--quiet'],
|
||||
25_000,
|
||||
);
|
||||
if (cap.code !== 0) {
|
||||
// The IRON RULE is "does not hang." A non-zero exit that still returned
|
||||
// within the window is tolerable in keyless CI; a hang (kill at 25s) is not.
|
||||
expect(cap.durationMs).toBeLessThan(25_000);
|
||||
} else {
|
||||
expect(cap.code).toBe(0);
|
||||
}
|
||||
|
||||
// The real lock-pin symptom: the NEXT command times out waiting for the
|
||||
// PGLite lock. Assert a subsequent read runs cleanly and quickly.
|
||||
const next = await runWithTimeout(['get', 'meetings/capture-test'], 15_000);
|
||||
expect(next.durationMs).toBeLessThan(15_000);
|
||||
expect(next.stderr).not.toContain('Timed out waiting for PGLite lock');
|
||||
if (next.code === 0) {
|
||||
expect(next.stdout.toLowerCase()).toContain('foxtrot');
|
||||
}
|
||||
}, 60_000);
|
||||
});
|
||||
|
||||
describe('v0.41.8.0 — daemon survival (regression guard for narrow force-exit)', () => {
|
||||
test('gbrain serve --http stays alive past the timeout window', async () => {
|
||||
// Pick a likely-free ephemeral port. We're testing "still alive
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
/**
|
||||
* E2E for #1745 (v0.42.20.0): module-mode reconnect() must NOT tear down the
|
||||
* shared module singleton.
|
||||
*
|
||||
* The bug: a transient blip triggers withRetry's reconnect callback →
|
||||
* PostgresEngine.reconnect() → this.disconnect() → (module mode) db.disconnect()
|
||||
* → `sql.end(); sql = null`. Concurrent ops (other dream-cycle phases, the
|
||||
* minion-queue promoteDelayed loop) read db.getConnection() during that null
|
||||
* window and throw "No database connection: connect() has not been called".
|
||||
*
|
||||
* The fix: module-mode reconnect() never calls db.disconnect(); it idempotently
|
||||
* re-establishes via db.connect() (a no-op when the singleton is alive, which is
|
||||
* the common case) and refreshes the ConnectionManager read pool. postgres.js
|
||||
* auto-heals dead sockets, so a transient blip recovers without a teardown.
|
||||
*
|
||||
* Instance-pool engines (worker / `jobs work`) keep the teardown+recreate path.
|
||||
*/
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { PostgresEngine } from '../../src/core/postgres-engine.ts';
|
||||
import * as db from '../../src/core/db.ts';
|
||||
|
||||
const DATABASE_URL = process.env.DATABASE_URL;
|
||||
const skip = !DATABASE_URL;
|
||||
|
||||
if (skip) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('Skipping postgres-reconnect-singleton E2E (DATABASE_URL not set)');
|
||||
}
|
||||
|
||||
describe.skipIf(skip)('#1745 — module-mode reconnect preserves the shared singleton', () => {
|
||||
beforeAll(async () => {
|
||||
await db.disconnect();
|
||||
await db.connect({ database_url: DATABASE_URL! });
|
||||
}, 30_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await db.disconnect();
|
||||
});
|
||||
|
||||
test('module reconnect() keeps db.getConnection() live (no null window)', async () => {
|
||||
const engine = new PostgresEngine();
|
||||
await engine.connect({ database_url: DATABASE_URL! }); // module singleton (no poolSize)
|
||||
|
||||
// Capture the singleton reference BEFORE reconnect — pre-fix, reconnect's
|
||||
// disconnect would sql.end() + null this out.
|
||||
const sqlBefore = db.getConnection();
|
||||
|
||||
await engine.reconnect();
|
||||
|
||||
// Post-fix: the singleton is untouched (db.connect() is a no-op when alive),
|
||||
// so getConnection() still returns a usable client and queries still work.
|
||||
const sqlAfter = db.getConnection();
|
||||
const rows = await sqlAfter.unsafe('SELECT 1 as ok');
|
||||
expect((rows[0] as unknown as { ok: number }).ok).toBe(1);
|
||||
// Same live pool object (no teardown/recreate in module mode).
|
||||
expect(sqlAfter).toBe(sqlBefore);
|
||||
}, 30_000);
|
||||
|
||||
test('concurrent reader during reconnect never sees "connect() has not been called"', async () => {
|
||||
const engine = new PostgresEngine();
|
||||
await engine.connect({ database_url: DATABASE_URL! });
|
||||
|
||||
// Fire a burst of reads through the module singleton WHILE reconnecting.
|
||||
// Pre-fix, the disconnect→connect window nulled the singleton and the
|
||||
// concurrent readers threw. Post-fix there is no null window.
|
||||
const readers = Array.from({ length: 20 }, async () => {
|
||||
const r = await db.getConnection().unsafe('SELECT 1 as ok');
|
||||
return (r[0] as unknown as { ok: number }).ok;
|
||||
});
|
||||
const [reconnectErr, ...results] = await Promise.all([
|
||||
engine.reconnect().then(() => null).catch((e) => e),
|
||||
...readers,
|
||||
]);
|
||||
expect(reconnectErr).toBeNull();
|
||||
for (const ok of results) expect(ok).toBe(1);
|
||||
}, 30_000);
|
||||
|
||||
test('instance-pool reconnect() still teardown+recreates its own pool', async () => {
|
||||
const engine = new PostgresEngine();
|
||||
await engine.connect({ database_url: DATABASE_URL!, poolSize: 2 });
|
||||
|
||||
// Works before.
|
||||
const before = await engine.executeRaw<{ ok: number }>('SELECT 1 as ok');
|
||||
expect(before[0].ok).toBe(1);
|
||||
|
||||
await engine.reconnect(); // instance path: tears down + rebuilds _sql
|
||||
|
||||
// Works after (fresh pool).
|
||||
const after = await engine.executeRaw<{ ok: number }>('SELECT 1 as ok');
|
||||
expect(after[0].ok).toBe(1);
|
||||
|
||||
await engine.disconnect();
|
||||
|
||||
// Module singleton untouched by the instance engine's reconnect/disconnect.
|
||||
const mod = await db.getConnection().unsafe('SELECT 1 as ok');
|
||||
expect((mod[0] as unknown as { ok: number }).ok).toBe(1);
|
||||
}, 30_000);
|
||||
});
|
||||
@@ -0,0 +1,77 @@
|
||||
/**
|
||||
* v0.42.20.0 (#1762) — eval-capture is the 4th fire-and-forget DB-write sink the
|
||||
* background-work registry drains before CLI disconnect. `captureEvalCandidate`
|
||||
* is `void`-ed by the search/query op handlers; its async `logEvalCandidate`
|
||||
* write is the same lock-pin / disconnect-race class as the other sinks. These
|
||||
* tests pin the bounded drain (`awaitPendingEvalCaptures`).
|
||||
*/
|
||||
import { describe, test, expect, afterEach } from 'bun:test';
|
||||
import {
|
||||
captureEvalCandidate,
|
||||
awaitPendingEvalCaptures,
|
||||
_resetPendingEvalCapturesForTests,
|
||||
type CaptureContext,
|
||||
} from '../src/core/eval-capture.ts';
|
||||
import type { BrainEngine } from '../src/core/engine.ts';
|
||||
import type { SearchResult } from '../src/core/types.ts';
|
||||
|
||||
function makeCtx(): CaptureContext {
|
||||
const result: SearchResult = {
|
||||
slug: 'people/alice-example',
|
||||
page_id: 1,
|
||||
title: 'Alice Example',
|
||||
type: 'person',
|
||||
chunk_text: '…',
|
||||
chunk_source: 'compiled_truth',
|
||||
chunk_id: 42,
|
||||
chunk_index: 0,
|
||||
score: 0.9,
|
||||
stale: false,
|
||||
source_id: 'default',
|
||||
};
|
||||
return {
|
||||
tool_name: 'query',
|
||||
query: 'who is alice',
|
||||
results: [result],
|
||||
meta: { vector_enabled: true, detail_resolved: 'medium', expansion_applied: false },
|
||||
latency_ms: 1,
|
||||
remote: false,
|
||||
expand_enabled: false,
|
||||
detail: null,
|
||||
job_id: null,
|
||||
subagent_id: null,
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(() => _resetPendingEvalCapturesForTests());
|
||||
|
||||
describe('awaitPendingEvalCaptures', () => {
|
||||
test('empty set drains instantly', async () => {
|
||||
const r = await awaitPendingEvalCaptures(50);
|
||||
expect(r.unfinished).toBe(0);
|
||||
});
|
||||
|
||||
test('drains a settled capture to unfinished:0', async () => {
|
||||
const engine = {
|
||||
logEvalCandidate: async () => 1,
|
||||
logEvalCaptureFailure: async () => {},
|
||||
} as unknown as BrainEngine;
|
||||
void captureEvalCandidate(engine, makeCtx(), { scrub_pii: false });
|
||||
const r = await awaitPendingEvalCaptures(1000);
|
||||
expect(r.unfinished).toBe(0);
|
||||
});
|
||||
|
||||
test('a hanging capture is bounded by the timeout (not a hang)', async () => {
|
||||
const engine = {
|
||||
// Never resolves — simulates a wedged DB write.
|
||||
logEvalCandidate: () => new Promise<number>(() => {}),
|
||||
logEvalCaptureFailure: async () => {},
|
||||
} as unknown as BrainEngine;
|
||||
void captureEvalCandidate(engine, makeCtx(), { scrub_pii: false });
|
||||
const start = Date.now();
|
||||
const r = await awaitPendingEvalCaptures(150);
|
||||
const elapsed = Date.now() - start;
|
||||
expect(r.unfinished).toBe(1);
|
||||
expect(elapsed).toBeLessThan(1000);
|
||||
});
|
||||
});
|
||||
@@ -20,19 +20,19 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { readFileSync } from 'fs';
|
||||
|
||||
describe('v0.36.1.x #1125 — query drain cache writes before CLI exit', () => {
|
||||
test("cli.ts awaits awaitPendingSearchCacheWrites for the 'query' op", () => {
|
||||
const src = readFileSync('src/cli.ts', 'utf8');
|
||||
// Sequence: 'query' op match → import the drain → await
|
||||
expect(src).toMatch(/op\.name\s*===\s*'query'[\s\S]{0,200}awaitPendingSearchCacheWrites/);
|
||||
expect(src).toMatch(/await\s+awaitPendingSearchCacheWrites\(\)/);
|
||||
});
|
||||
|
||||
test('hybrid.ts exports the drain helper + trackCacheWrite', () => {
|
||||
describe('v0.42.20.0 — search-cache drained via the background-work registry', () => {
|
||||
// Supersedes the v0.36.1.x #1125 query-only drain: search-cache now registers
|
||||
// a registry drainer (drained for BOTH search and query, bounded), and cli.ts
|
||||
// drains the whole registry rather than calling awaitPendingSearchCacheWrites
|
||||
// directly for the 'query' op only.
|
||||
test('hybrid.ts registers a bounded search-cache drainer', () => {
|
||||
const src = readFileSync('src/core/search/hybrid.ts', 'utf8');
|
||||
expect(src).toMatch(/export async function awaitPendingSearchCacheWrites/);
|
||||
expect(src).toMatch(/pendingCacheWrites\.add\(promise\)/);
|
||||
expect(src).toMatch(/trackCacheWrite\(/);
|
||||
// Now bounded (was an unbounded Promise.allSettled) + registered.
|
||||
expect(src).toMatch(/registerBackgroundWorkDrainer\(\{[\s\S]*?name:\s*'search-cache'/);
|
||||
expect(src).toMatch(/Promise\.race/);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -123,51 +123,58 @@ describe('v0.36.1.x #1124 — query --no-expand actually negates expand', () =>
|
||||
});
|
||||
});
|
||||
|
||||
describe('v0.41.8.0 #1247/#1269/#1290 — drain last-retrieved before CLI disconnect', () => {
|
||||
test('cli.ts imports awaitPendingLastRetrievedWrites', () => {
|
||||
describe('v0.42.20.0 — background-work registry drains every sink before disconnect', () => {
|
||||
// Supersedes the v0.41.8.0 #1247/#1269/#1290 per-call last-retrieved drain:
|
||||
// last-retrieved is now one of four registry sinks; cli.ts drains the whole
|
||||
// registry (drainAllBackgroundWorkForCliExit) before disconnect on BOTH the
|
||||
// op-dispatch path AND the CLI_ONLY path (the latter closes #1762 for capture).
|
||||
test('cli.ts imports + uses drainAllBackgroundWorkForCliExit', () => {
|
||||
const src = readFileSync('src/cli.ts', 'utf8');
|
||||
// Allow additional type-imports from the same module (e.g. `type DrainOutcome`)
|
||||
expect(src).toMatch(/import\s+\{[^}]*\bawaitPendingLastRetrievedWrites\b[^}]*\}\s*from\s+['"]\.\/core\/last-retrieved\.ts['"]/);
|
||||
expect(src).toMatch(/import\s+\{\s*drainAllBackgroundWorkForCliExit\s*\}\s*from\s+['"]\.\/core\/background-work\.ts['"]/);
|
||||
// Two call sites: op-dispatch finally + handleCliOnly finally.
|
||||
const calls = src.match(/await\s+drainAllBackgroundWorkForCliExit\s*\(/g) ?? [];
|
||||
expect(calls.length).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
test('last-retrieved.ts exports the drain + tracks promises in a module-scoped Set', () => {
|
||||
test('last-retrieved.ts still exports the bounded drain + registers a drainer', () => {
|
||||
const src = readFileSync('src/core/last-retrieved.ts', 'utf8');
|
||||
expect(src).toMatch(/export async function awaitPendingLastRetrievedWrites/);
|
||||
expect(src).toMatch(/pendingLastRetrievedWrites\s*=\s*new\s+Set/);
|
||||
expect(src).toMatch(/pendingLastRetrievedWrites\.add\(promise\)/);
|
||||
// Per D5+D8: snapshot pattern (Codex finding #3) + bounded timeout
|
||||
expect(src).toMatch(/Promise\.race/);
|
||||
expect(src).toMatch(/drain timed out/);
|
||||
expect(src).toMatch(/registerBackgroundWorkDrainer\(\{[\s\S]*?name:\s*'last-retrieved'/);
|
||||
});
|
||||
|
||||
test('cli.ts behavioral positioning: drain appears BEFORE engine.disconnect in op-dispatch', () => {
|
||||
test('all four sinks register a drainer', () => {
|
||||
expect(readFileSync('src/core/facts/queue.ts', 'utf8'))
|
||||
.toMatch(/registerBackgroundWorkDrainer\(\{[\s\S]*?name:\s*'facts'[\s\S]*?abort:/);
|
||||
expect(readFileSync('src/core/search/hybrid.ts', 'utf8'))
|
||||
.toMatch(/name:\s*'search-cache'/);
|
||||
expect(readFileSync('src/core/last-retrieved.ts', 'utf8'))
|
||||
.toMatch(/name:\s*'last-retrieved'/);
|
||||
expect(readFileSync('src/core/eval-capture.ts', 'utf8'))
|
||||
.toMatch(/name:\s*'eval-capture'/);
|
||||
});
|
||||
|
||||
test('cli.ts behavioral positioning: registry drain appears BEFORE engine.disconnect (op-dispatch)', () => {
|
||||
const src = readFileSync('src/cli.ts', 'utf8');
|
||||
// Per D5+D8: replaces the brittle literal-output regex from PR #1259
|
||||
// with a behavioral-positioning assertion. The drain CALL must appear
|
||||
// textually before the disconnect CALL in the local-engine path.
|
||||
// Match `await fn(` not bare names — bare names also appear in
|
||||
// comments and would false-match the comment ordering.
|
||||
const localPath = src.match(/\/\/ Local engine path \(unchanged behavior[\s\S]+?^\}/m);
|
||||
expect(localPath).not.toBeNull();
|
||||
const block = localPath![0];
|
||||
const drainCallRe = /await\s+awaitPendingLastRetrievedWrites\s*\(/;
|
||||
const drainCallRe = /await\s+drainAllBackgroundWorkForCliExit\s*\(/;
|
||||
const disconnectCallRe = /await\s+engine\.disconnect\s*\(/;
|
||||
expect(block).toMatch(drainCallRe);
|
||||
expect(block).toMatch(disconnectCallRe);
|
||||
const drainMatch = block.match(drainCallRe);
|
||||
const disconnectMatch = block.match(disconnectCallRe);
|
||||
const drainIdx = block.indexOf(drainMatch![0]);
|
||||
const disconnectIdx = block.indexOf(disconnectMatch![0]);
|
||||
expect(drainIdx).toBeGreaterThan(-1);
|
||||
expect(disconnectIdx).toBeGreaterThan(-1);
|
||||
const drainIdx = block.indexOf(block.match(drainCallRe)![0]);
|
||||
const disconnectIdx = block.indexOf(block.match(disconnectCallRe)![0]);
|
||||
expect(drainIdx).toBeLessThan(disconnectIdx);
|
||||
});
|
||||
|
||||
test('cli.ts uses shouldForceExitAfterMain only on the timeout path', () => {
|
||||
const src = readFileSync('src/cli.ts', 'utf8');
|
||||
expect(src).toMatch(/import\s+\{\s*shouldForceExitAfterMain\s*\}\s*from\s+['"]\.\/core\/cli-force-exit\.ts['"]/);
|
||||
// The force-exit gate MUST be conditioned on drainResult.outcome ==='timeout'
|
||||
expect(src).toMatch(/drainResult\.outcome\s*===\s*['"]timeout['"]/);
|
||||
test('background-work.ts: Map registry, ordered drain, awaited abort, test seam', () => {
|
||||
const src = readFileSync('src/core/background-work.ts', 'utf8');
|
||||
expect(src).toMatch(/new\s+Map<string,\s*BackgroundWorkDrainer>/);
|
||||
expect(src).toMatch(/sort\(\s*\(a,\s*b\)\s*=>\s*a\.order\s*-\s*b\.order/);
|
||||
expect(src).toMatch(/if\s*\(unfinished\s*>\s*0\s*&&\s*d\.abort\)\s*\{[\s\S]*?await\s+d\.abort\(\)/);
|
||||
expect(src).toMatch(/export function __registerDrainerForTest/);
|
||||
});
|
||||
|
||||
test('cli-force-exit.ts daemon guard excludes "serve"', () => {
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
/**
|
||||
* v0.42.20.0 (Fix 3, #1775) — query-embed deadline unit tests.
|
||||
*
|
||||
* The regression: `search`/`query` default to cheap-hybrid, which embeds the
|
||||
* query. A stalled embedding provider made the embed `await` never settle, so
|
||||
* the handler never reached the keyword fallback and the CLI force-exited at 10s
|
||||
* with no output. `embedQueryBounded` bounds the embed so it THROWS on timeout
|
||||
* → the caller's existing try/catch falls back to keyword.
|
||||
*
|
||||
* These tests prove the bound fires even when the transport IGNORES the
|
||||
* abortSignal (the Promise.race guarantee — codex #3: abortSignal alone is
|
||||
* insufficient against a wedged provider), and that a shared/elapsed deadline
|
||||
* makes a second embed fail FAST (worst case ~one timeout, not two).
|
||||
*/
|
||||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import {
|
||||
configureGateway,
|
||||
resetGateway,
|
||||
__setEmbedTransportForTests,
|
||||
} from '../../src/core/ai/gateway.ts';
|
||||
import { embedQueryBounded, makeQueryEmbedDeadline } from '../../src/core/search/hybrid.ts';
|
||||
|
||||
describe('embedQueryBounded — query-embed deadline', () => {
|
||||
beforeEach(() => {
|
||||
resetGateway();
|
||||
configureGateway({
|
||||
embedding_model: 'voyage:voyage-4-large',
|
||||
embedding_dimensions: 1024,
|
||||
env: { VOYAGE_API_KEY: 'voyage-fake' },
|
||||
});
|
||||
});
|
||||
afterEach(() => {
|
||||
__setEmbedTransportForTests(null);
|
||||
resetGateway();
|
||||
});
|
||||
|
||||
test('rejects within the budget when the transport hangs (ignores abort)', async () => {
|
||||
// Transport never resolves AND ignores the abort signal — only the
|
||||
// Promise.race deadline can save us.
|
||||
__setEmbedTransportForTests(() => new Promise(() => { /* hang forever */ }));
|
||||
const dl = makeQueryEmbedDeadline(200);
|
||||
const start = Date.now();
|
||||
let threw = false;
|
||||
try {
|
||||
await embedQueryBounded('locker code', undefined, dl);
|
||||
} catch (e) {
|
||||
threw = true;
|
||||
expect(String((e as Error).message)).toContain('deadline');
|
||||
}
|
||||
const elapsed = Date.now() - start;
|
||||
expect(threw).toBe(true);
|
||||
// The 200ms deadline is floored to MIN_QUERY_EMBED_BUDGET_MS (2s) — the bound
|
||||
// still fires (not infinite hang), comfortably under the 10s CLI force-exit.
|
||||
expect(elapsed).toBeLessThan(3000);
|
||||
});
|
||||
|
||||
test('an already-elapsed shared deadline is floored, not fresh-6s (codex floor)', async () => {
|
||||
__setEmbedTransportForTests(() => new Promise(() => { /* hang forever */ }));
|
||||
// Simulate the inner embed reusing a deadline the cache-lookup already spent.
|
||||
const dl = { signal: AbortSignal.timeout(1), deadlineAt: Date.now() - 5 };
|
||||
const start = Date.now();
|
||||
let threw = false;
|
||||
try {
|
||||
await embedQueryBounded('q', undefined, dl);
|
||||
} catch {
|
||||
threw = true;
|
||||
}
|
||||
const elapsed = Date.now() - start;
|
||||
expect(threw).toBe(true);
|
||||
// Floored to MIN_QUERY_EMBED_BUDGET_MS (2s) — NOT a fresh 6s (would blow the
|
||||
// cached-path total past the 10s force-exit) and NOT ~0 (would starve a
|
||||
// healthy inner embed). So: rejects after ~2s, comfortably under 6s.
|
||||
expect(elapsed).toBeGreaterThanOrEqual(1800);
|
||||
expect(elapsed).toBeLessThan(3500);
|
||||
});
|
||||
|
||||
test('resolves with the embedding when the transport returns in time', async () => {
|
||||
const vec = Array.from({ length: 1024 }, () => 0.1);
|
||||
__setEmbedTransportForTests(async () => ({ embeddings: [vec], usage: { tokens: 1 } }) as any);
|
||||
const dl = makeQueryEmbedDeadline(2000);
|
||||
const out = await embedQueryBounded('q', undefined, dl);
|
||||
expect(out).toBeInstanceOf(Float32Array);
|
||||
expect(out.length).toBe(1024);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user