mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-30 03:12:32 +00:00
Two infrastructure modules the subagent handler spine depends on:
rate-leases.ts — lease-based concurrency cap for outbound providers
(anthropic:messages, openai:*, etc.). Counter-based limiters leak capacity
on worker crash; leases are owner-tagged rows with expires_at that
auto-prune on the next acquire. Two-phase: txn-scoped pg_advisory_xact_lock
guards the check-then-insert so concurrent acquires can't both win the
"last slot". renewLeaseWithBackoff retries 3x (250/500/1000ms) for mid-
call DB blips — on persistent failure the LLM-loop caller aborts with a
renewable error so the worker re-claims and the rate invariant is
preserved. Owner FK cascades clean up leases on job deletion.
wait-for-completion.ts — poll-until-terminal helper for CLI callers.
Minions' NOTIFY is worker-side only; `gbrain agent run --follow` polls
getJob() until status is {completed, failed, dead, cancelled}. TimeoutError
carries jobId + elapsedMs and does NOT cancel the job — the user can
inspect via `gbrain jobs get <id>` later. Supports AbortSignal for Ctrl-C
without throwing. Default pollMs is 1000 on Postgres, 250 on PGLite (inline
CLI has no network RTT).
21 unit tests cover: single/multi acquire under cap, rejection past cap,
release frees slot, different keys are independent, stale prune, cascade
on owner delete, renew bumps expires_at, renew on missing is false,
backoff path success + pruned short-circuit. waitForCompletion: fast-path
terminal, transitions mid-wait (completed/failed/cancelled), TimeoutError
shape, abort-signal early exit, non-existent job error.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
113 lines
3.9 KiB
TypeScript
113 lines
3.9 KiB
TypeScript
/**
|
|
* waitForCompletion tests. Uses PGLite in-memory so the poll path exercises
|
|
* a real getJob over a real engine.
|
|
*/
|
|
|
|
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
|
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
|
import { MinionQueue } from '../src/core/minions/queue.ts';
|
|
import { waitForCompletion, TimeoutError, __testing } from '../src/core/minions/wait-for-completion.ts';
|
|
|
|
let engine: PGLiteEngine;
|
|
let queue: MinionQueue;
|
|
|
|
beforeAll(async () => {
|
|
engine = new PGLiteEngine();
|
|
await engine.connect({ databaseUrl: '' });
|
|
await engine.initSchema();
|
|
queue = new MinionQueue(engine);
|
|
});
|
|
|
|
afterAll(async () => {
|
|
await engine.disconnect();
|
|
});
|
|
|
|
beforeEach(async () => {
|
|
await engine.executeRaw('DELETE FROM minion_jobs');
|
|
});
|
|
|
|
describe('waitForCompletion terminal states', () => {
|
|
test('TERMINAL_STATES covers every terminal MinionJobStatus value', () => {
|
|
expect(__testing.TERMINAL_STATES).toEqual(['completed', 'failed', 'dead', 'cancelled']);
|
|
});
|
|
|
|
test('returns immediately when job already completed (fast path)', async () => {
|
|
const j = await queue.add('t', {});
|
|
const claimed = await queue.claim('tok', 30000, 'default', ['t']);
|
|
await queue.completeJob(claimed!.id, 'tok', { ok: true });
|
|
|
|
const t0 = Date.now();
|
|
const res = await waitForCompletion(queue, j.id, { pollMs: 500 });
|
|
expect(res.status).toBe('completed');
|
|
expect(Date.now() - t0).toBeLessThan(300); // no full poll cycle
|
|
});
|
|
|
|
test('returns when job transitions to failed mid-wait', async () => {
|
|
const j = await queue.add('t', {});
|
|
const p = waitForCompletion(queue, j.id, { pollMs: 25, timeoutMs: 5000 });
|
|
// Transition the job to failed after a brief delay.
|
|
setTimeout(async () => {
|
|
const claimed = await queue.claim('tok', 30000, 'default', ['t']);
|
|
await queue.failJob(claimed!.id, 'tok', 'boom', 'failed');
|
|
}, 60);
|
|
const res = await p;
|
|
expect(res.status).toBe('failed');
|
|
});
|
|
|
|
test('returns when job transitions to cancelled', async () => {
|
|
const j = await queue.add('t', {});
|
|
const p = waitForCompletion(queue, j.id, { pollMs: 25, timeoutMs: 5000 });
|
|
setTimeout(() => { queue.cancelJob(j.id); }, 60);
|
|
const res = await p;
|
|
expect(res.status).toBe('cancelled');
|
|
});
|
|
|
|
test('throws TimeoutError when job stays non-terminal past timeoutMs', async () => {
|
|
const j = await queue.add('t', {});
|
|
await expect(
|
|
waitForCompletion(queue, j.id, { pollMs: 25, timeoutMs: 100 })
|
|
).rejects.toBeInstanceOf(TimeoutError);
|
|
});
|
|
|
|
test('TimeoutError carries the jobId and elapsedMs', async () => {
|
|
const j = await queue.add('t', {});
|
|
try {
|
|
await waitForCompletion(queue, j.id, { pollMs: 25, timeoutMs: 80 });
|
|
throw new Error('should have thrown');
|
|
} catch (e) {
|
|
expect(e).toBeInstanceOf(TimeoutError);
|
|
const te = e as TimeoutError;
|
|
expect(te.jobId).toBe(j.id);
|
|
expect(te.elapsedMs).toBeGreaterThanOrEqual(80);
|
|
}
|
|
});
|
|
|
|
test('TimeoutError does NOT cancel the job', async () => {
|
|
const j = await queue.add('t', {});
|
|
try {
|
|
await waitForCompletion(queue, j.id, { pollMs: 25, timeoutMs: 80 });
|
|
} catch {}
|
|
const still = await queue.getJob(j.id);
|
|
expect(still?.status).toBe('waiting');
|
|
});
|
|
|
|
test('AbortSignal exits loop early without throwing', async () => {
|
|
const j = await queue.add('t', {});
|
|
const ac = new AbortController();
|
|
setTimeout(() => ac.abort(), 50);
|
|
const res = await waitForCompletion(queue, j.id, {
|
|
pollMs: 25,
|
|
timeoutMs: 5000,
|
|
signal: ac.signal,
|
|
});
|
|
expect(res.id).toBe(j.id);
|
|
// Still waiting — we just stopped polling.
|
|
expect(res.status).toBe('waiting');
|
|
});
|
|
|
|
test('throws when job id does not exist', async () => {
|
|
await expect(waitForCompletion(queue, 99_999, { pollMs: 10, timeoutMs: 100 }))
|
|
.rejects.toThrow(/not found/);
|
|
});
|
|
});
|