import { describe, it, expect, afterEach } from 'bun:test'; import { existsSync, readFileSync, writeFileSync, unlinkSync, chmodSync, mkdirSync, rmSync } from 'fs'; import { spawn } from 'child_process'; import { join } from 'path'; import { tmpdir } from 'os'; import { readSupervisorEvents, computeSupervisorAuditFilename } from '../src/core/minions/handlers/supervisor-audit.ts'; import { calculateBackoffMs, resolveHardStopMaxCrashes } from '../src/core/minions/supervisor.ts'; const TEST_PID_FILE = '/tmp/gbrain-supervisor-test.pid'; afterEach(() => { try { unlinkSync(TEST_PID_FILE); } catch { /* noop */ } }); // ----- Integration test helpers ----- interface IntegrationHarness { pidFile: string; auditDir: string; workerScript: string; envOutFile: string; cleanup: () => void; } /** Create per-test temp files + a fake worker shell script. */ function makeHarness(name: string, workerBody: string): IntegrationHarness { const tmpRoot = join(tmpdir(), `gbrain-sup-test-${name}-${process.pid}-${Date.now()}`); mkdirSync(tmpRoot, { recursive: true }); const pidFile = join(tmpRoot, 'supervisor.pid'); const auditDir = join(tmpRoot, 'audit'); const workerScript = join(tmpRoot, 'worker.sh'); const envOutFile = join(tmpRoot, 'env-out.txt'); writeFileSync(workerScript, `#!/bin/sh\n${workerBody}\n`, 'utf8'); chmodSync(workerScript, 0o755); return { pidFile, auditDir, workerScript, envOutFile, cleanup: () => { try { rmSync(tmpRoot, { recursive: true, force: true }); } catch { /* noop */ } }, }; } /** * Spawn the supervisor runner as a subprocess. Returns a handle with the * child, a promise resolving to exit code + signal, and a kill helper. */ function spawnSupervisor(h: IntegrationHarness, overrides: Record = {}) { const env: Record = { ...(process.env as Record), SUP_PID_FILE: h.pidFile, SUP_CLI_PATH: h.workerScript, SUP_AUDIT_DIR: h.auditDir, SUP_BACKOFF_FLOOR_MS: '5', SUP_MAX_CRASHES: '3', SUP_HEALTH_INTERVAL_MS: '999999', // effectively off ...overrides, }; // issue #1994: the soft crash budget now DEGRADES (retry-with-backoff) rather // than permanently giving up; permanent give-up fires at a much-higher hard // ceiling (maxCrashes × 10). These integration tests assert the give-up // LIFECYCLE (audit events, exit code, pidfile cleanup), so pin the hard // ceiling to the soft budget by default — the degraded path is unit-tested in // child-worker-supervisor.test.ts. Tests that want true degraded behavior // pass GBRAIN_SUPERVISOR_HARD_STOP_CRASHES explicitly. if (env.GBRAIN_SUPERVISOR_HARD_STOP_CRASHES === undefined) { env.GBRAIN_SUPERVISOR_HARD_STOP_CRASHES = env.SUP_MAX_CRASHES; } const child = spawn('bun', [join(import.meta.dir, 'fixtures/supervisor-runner.ts')], { env, stdio: ['ignore', 'pipe', 'pipe'], }); let stdout = ''; let stderr = ''; child.stdout?.on('data', (d) => { stdout += d.toString(); }); child.stderr?.on('data', (d) => { stderr += d.toString(); }); const exited = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve) => { child.on('exit', (code, signal) => resolve({ code, signal })); }); return { child, exited, getStdout: () => stdout, getStderr: () => stderr, }; } /** Read the audit JSONL for the current week. */ function readAudit(auditDir: string) { const origEnv = process.env.GBRAIN_AUDIT_DIR; process.env.GBRAIN_AUDIT_DIR = auditDir; try { return readSupervisorEvents(); } finally { if (origEnv === undefined) delete process.env.GBRAIN_AUDIT_DIR; else process.env.GBRAIN_AUDIT_DIR = origEnv; } } /** Poll until predicate returns true or deadline elapses. */ async function waitFor(pred: () => boolean, timeoutMs: number, tickMs = 20): Promise { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { if (pred()) return true; await new Promise(r => setTimeout(r, tickMs)); } return pred(); } describe('MinionSupervisor', () => { describe('resolveHardStopMaxCrashes (issue #1994)', () => { const KEY = 'GBRAIN_SUPERVISOR_HARD_STOP_CRASHES'; afterEach(() => { delete process.env[KEY]; }); it('defaults to maxCrashes × 10 when no override', () => { delete process.env[KEY]; expect(resolveHardStopMaxCrashes(10)).toBe(100); expect(resolveHardStopMaxCrashes(3)).toBe(30); }); it('honors a valid non-negative integer override', () => { process.env[KEY] = '0'; // 0 = disable permanent give-up expect(resolveHardStopMaxCrashes(10)).toBe(0); process.env[KEY] = '5'; expect(resolveHardStopMaxCrashes(10)).toBe(5); }); it('ignores a negative or non-integer override (falls back to default)', () => { process.env[KEY] = '-1'; expect(resolveHardStopMaxCrashes(10)).toBe(100); process.env[KEY] = 'abc'; expect(resolveHardStopMaxCrashes(10)).toBe(100); }); }); describe('calculateBackoffMs', () => { it('returns ~1s for first crash', () => { const backoff = calculateBackoffMs(0); expect(backoff).toBeGreaterThanOrEqual(1000); expect(backoff).toBeLessThan(1200); // 1000 + 10% jitter max }); it('doubles with each crash', () => { const b0 = calculateBackoffMs(0); const b1 = calculateBackoffMs(1); const b2 = calculateBackoffMs(2); // Approximate: b1 should be ~2x b0, b2 ~2x b1 (within jitter) expect(b1).toBeGreaterThan(1800); expect(b2).toBeGreaterThan(3600); }); it('caps at 60s', () => { const backoff = calculateBackoffMs(20); // 2^20 * 1000 would be huge expect(backoff).toBeLessThanOrEqual(66_000); // 60s + 10% jitter }); it('includes jitter (not perfectly deterministic)', () => { const values = new Set(); for (let i = 0; i < 10; i++) { values.add(Math.round(calculateBackoffMs(3))); } // With 10% jitter, we should get some variation expect(values.size).toBeGreaterThan(1); }); }); describe('PID file management', () => { it('detects stale PID files', () => { // Write a PID file with a non-existent PID writeFileSync(TEST_PID_FILE, '999999999'); expect(existsSync(TEST_PID_FILE)).toBe(true); // A real supervisor would detect this as stale and overwrite const existingPid = parseInt(readFileSync(TEST_PID_FILE, 'utf8').trim(), 10); let isAlive = false; try { process.kill(existingPid, 0); isAlive = true; } catch { isAlive = false; } expect(isAlive).toBe(false); }); it('detects live PID files (current process)', () => { // Write our own PID writeFileSync(TEST_PID_FILE, String(process.pid)); const existingPid = parseInt(readFileSync(TEST_PID_FILE, 'utf8').trim(), 10); let isAlive = false; try { process.kill(existingPid, 0); isAlive = true; } catch { isAlive = false; } expect(isAlive).toBe(true); expect(existingPid).toBe(process.pid); }); }); describe('crash count tracking', () => { it('backoff escalates with crash count', () => { const backoffs = []; for (let i = 0; i < 7; i++) { backoffs.push(calculateBackoffMs(i)); } // Each should be roughly 2x the previous (within jitter) for (let i = 1; i < 6; i++) { // The base doubles, so even with jitter the next should be > 1.5x previous expect(backoffs[i]).toBeGreaterThan(backoffs[i - 1] * 1.5); } }); }); // -------------------------------------------------------------- // Integration tests: real spawn(), real signals, real audit file. // Each test uses a unique tmpdir harness so they can run in parallel // without colliding. `_backoffFloorMs: 5` (set via SUP_BACKOFF_FLOOR_MS) // keeps the whole suite under a few seconds. // -------------------------------------------------------------- describe('integration: crash → restart → max-crashes lifecycle', () => { it('respawns the worker after a crash and eventually exits with max-crashes code=1', async () => { // Worker always exits with code 1; supervisor should respawn it 3 times, // hit max-crashes, then exit via shutdown() with code 1. const h = makeHarness('max-crashes', 'exit 1'); try { // hard ceiling defaults to SUP_MAX_CRASHES in the harness (see // spawnSupervisor) so this give-up lifecycle still fires at 3 (#1994). const sup = spawnSupervisor(h, { SUP_MAX_CRASHES: '3' }); const { code } = await sup.exited; expect(code).toBe(1); // PID file cleaned up on exit (synchronous process.on('exit') handler). expect(existsSync(h.pidFile)).toBe(false); // Audit file should contain started + 3x worker_spawned/worker_exited + // max_crashes_exceeded + shutting_down + stopped. const events = readAudit(h.auditDir); const eventTypes = events.map(e => e.event); expect(eventTypes).toContain('started'); expect(eventTypes.filter(t => t === 'worker_spawned').length).toBeGreaterThanOrEqual(3); expect(eventTypes.filter(t => t === 'worker_exited').length).toBeGreaterThanOrEqual(3); expect(eventTypes).toContain('max_crashes_exceeded'); expect(eventTypes).toContain('shutting_down'); expect(eventTypes).toContain('stopped'); // The stopped event should carry exit_code=1 and reason=max_crashes. const stoppedEvt = events.filter(e => e.event === 'stopped').pop(); expect((stoppedEvt as Record).exit_code).toBe(1); expect((stoppedEvt as Record).reason).toBe('max_crashes'); } finally { h.cleanup(); } }, 15_000); }); describe('integration: graceful SIGTERM during backoff', () => { it('receives SIGTERM while sleeping between crashes and exits 0 cleanly', async () => { // Worker always exits with code 1; supervisor has a high max-crashes // and a long-enough backoff floor that we can reliably catch it mid-sleep. const h = makeHarness('sigterm-backoff', 'exit 1'); try { const sup = spawnSupervisor(h, { SUP_MAX_CRASHES: '100', SUP_BACKOFF_FLOOR_MS: '800', // 800ms between restarts — enough to catch }); // Wait until the supervisor has written the PID file AND survived at // least one worker_exited (so it's definitely in the backoff sleep). const ready = await waitFor(() => { if (!existsSync(h.pidFile)) return false; const events = readAudit(h.auditDir); return events.some(e => e.event === 'worker_exited'); }, 3000); expect(ready).toBe(true); // Now SIGTERM the supervisor. It must exit cleanly within 200ms // (short-circuits the 800ms backoff sleep via the stopping flag). const sigSentAt = Date.now(); sup.child.kill('SIGTERM'); const { code, signal } = await sup.exited; const elapsed = Date.now() - sigSentAt; // Exit code 0 = clean; signal=null means we exited via process.exit, not got killed. expect(code).toBe(0); expect(signal).toBe(null); // Graceful, not hung: exit within 5s (process.exit() through shutdown() // should be near-instant; generous bound to tolerate CI slowness). expect(elapsed).toBeLessThan(5000); const events = readAudit(h.auditDir); const eventTypes = events.map(e => e.event); expect(eventTypes).toContain('shutting_down'); expect(eventTypes).toContain('stopped'); const shuttingEvt = events.filter(e => e.event === 'shutting_down').pop(); expect((shuttingEvt as Record).reason).toBe('SIGTERM'); // PID file cleaned up. expect(existsSync(h.pidFile)).toBe(false); } finally { h.cleanup(); } }, 20_000); }); describe('integration: env-var inheritance regression (codex #9 / eng #8)', () => { it('strips inherited GBRAIN_ALLOW_SHELL_JOBS when allowShellJobs=false, even if parent has it set', async () => { const outFile = join(tmpdir(), `gbrain-sup-env-${process.pid}-${Date.now()}.txt`); try { unlinkSync(outFile); } catch { /* may not exist */ } // Worker writes env to OUT_FILE then exits 1. exit=1 is required (not // exit=0) because post-D1/D2 (v0.33) clean exits don't count toward // crashCount — the supervisor would respawn forever. The test's // assertion is on the OUT_FILE contents (env plumbing), not the // exit code, so any non-zero code that trips SUP_MAX_CRASHES=1 works. const h = makeHarness('env-strip-outfile', `printf '%s\\n' "\${GBRAIN_ALLOW_SHELL_JOBS-UNSET}" > "$OUT_FILE" ; exit 1`); try { const sup = spawnSupervisor(h, { OUT_FILE: outFile, GBRAIN_ALLOW_SHELL_JOBS: '1', // parent has it SUP_ALLOW_SHELL_JOBS: '0', // supervisor says NO SUP_MAX_CRASHES: '1', }); await sup.exited; // Worker should have written "UNSET" (parent env var stripped from child). expect(existsSync(outFile)).toBe(true); const childSawEnv = readFileSync(outFile, 'utf8').trim(); expect(childSawEnv).toBe('UNSET'); } finally { try { unlinkSync(outFile); } catch { /* noop */ } h.cleanup(); } }, 15_000); it('DOES pass GBRAIN_ALLOW_SHELL_JOBS to child when allowShellJobs is true', async () => { const outFile = join(tmpdir(), `gbrain-sup-env-ok-${process.pid}-${Date.now()}.txt`); try { unlinkSync(outFile); } catch { /* may not exist */ } // Worker exits 1 (not 0) so SUP_MAX_CRASHES=1 actually trips. See // the comment on the env-strip test above for the v0.33 rationale. const h = makeHarness('env-pass-on-opt-in', `printf '%s\\n' "\${GBRAIN_ALLOW_SHELL_JOBS-UNSET}" > "$OUT_FILE" ; exit 1`); try { const sup = spawnSupervisor(h, { OUT_FILE: outFile, SUP_ALLOW_SHELL_JOBS: '1', SUP_MAX_CRASHES: '1', }); await sup.exited; expect(existsSync(outFile)).toBe(true); expect(readFileSync(outFile, 'utf8').trim()).toBe('1'); } finally { try { unlinkSync(outFile); } catch { /* noop */ } h.cleanup(); } }, 15_000); }); describe('integration: GBRAIN_SUPERVISED env var (v0.22.14)', () => { it('sets GBRAIN_SUPERVISED=1 on spawned worker child', async () => { const outFile = join(tmpdir(), `gbrain-sup-supervised-${process.pid}-${Date.now()}.txt`); try { unlinkSync(outFile); } catch { /* may not exist */ } // exit 1 required post-D1/D2 to trip SUP_MAX_CRASHES=1; clean exits // no longer count toward the crash limit. const h = makeHarness('supervised-env', `printf '%s\n' "\${GBRAIN_SUPERVISED-UNSET}" > "$OUT_FILE" ; exit 1`); try { const sup = spawnSupervisor(h, { OUT_FILE: outFile, SUP_MAX_CRASHES: '1', }); await sup.exited; expect(existsSync(outFile)).toBe(true); const childSawEnv = readFileSync(outFile, 'utf8').trim(); expect(childSawEnv).toBe('1'); } finally { try { unlinkSync(outFile); } catch { /* noop */ } h.cleanup(); } }, 15_000); }); describe('regression (R3): healthInterval=0 disables timer (v0.22.14)', () => { // Pre-fix: supervisor unconditionally called setInterval(callback, 0), // which schedules a tight loop on the next event-loop tick. The // operator-facing CLI claim "Use 0 to disable" was a lie — passing 0 // produced a DB-probe loop that hammered Postgres. // // Post-fix: setInterval is gated on healthInterval > 0. With 0, the // supervisor runs its supervise loop normally with the health timer // entirely absent. // // Assertion strategy: spawn the supervisor with SUP_HEALTH_INTERVAL_MS=0, // a fast worker that exits cleanly, and SUP_MAX_CRASHES=1. A working fix // should produce a single worker spawn → exit → supervisor shutdown // sequence. If the tight-loop bug returned, the supervisor would still // exit (max-crashes path) but the audit trail would show the tell-tale // signature of an extremely high health-check call rate during the brief // window before max-crashes fires. We assert the basic completion path // and let CI's wall-clock detect any pathological CPU spike. it('completes a normal supervise lifecycle with healthInterval=0', async () => { // exit 1 (not exit 0) because post-D1/D2 (v0.33) clean exits don't // count toward max_crashes — a code=0 worker would respawn forever. // The test's purpose is regression coverage that healthInterval=0 // disables the timer; the exit code doesn't matter to that assertion. const h = makeHarness('health-interval-zero', 'exit 1'); try { const sup = spawnSupervisor(h, { SUP_HEALTH_INTERVAL_MS: '0', SUP_MAX_CRASHES: '1', }); const start = Date.now(); const { code } = await sup.exited; const elapsedMs = Date.now() - start; // Clean exit (max-crashes path returns 1; this is fine — we just // want to confirm the supervisor reached its terminal state without // hanging or runaway looping). expect(code).toBe(1); // Sanity: a tight loop on setInterval(0) plus the spawn-respawn // loop would still terminate at max-crashes, but it would be // measurably slower than a clean run because the event loop is // saturated with health-check callbacks. Cap the upper bound at // 10s — clean runs typically finish in 1–2s. expect(elapsedMs).toBeLessThan(10_000); } finally { h.cleanup(); } }, 15_000); }); describe('integration: --max-rss spawn args (v0.21, auto-sized v0.41.39.0)', () => { it('passes an explicit --max-rss through to the spawned worker', async () => { const outFile = join(tmpdir(), `gbrain-sup-maxrss-${process.pid}-${Date.now()}.txt`); try { unlinkSync(outFile); } catch { /* may not exist */ } // SUP_MAX_RSS pins an explicit cap; the supervisor must pass it through // verbatim. exit 1 required post-D1/D2: code=0 workers respawn forever. const h = makeHarness('maxrss-explicit', `printf '%s\\n' "$*" > "$OUT_FILE" ; exit 1`); try { const sup = spawnSupervisor(h, { OUT_FILE: outFile, SUP_MAX_CRASHES: '1', SUP_MAX_RSS: '2048', }); await sup.exited; expect(existsSync(outFile)).toBe(true); const argv = readFileSync(outFile, 'utf8').trim(); expect(argv).toContain('--max-rss 2048'); } finally { try { unlinkSync(outFile); } catch { /* noop */ } h.cleanup(); } }, 15_000); // issue #1678: with no explicit cap the supervisor auto-sizes cgroup-aware // instead of the old flat 2048 footgun. Same machine → the in-test // resolveDefaultMaxRssMb() equals what the spawned supervisor computes. it('auto-sizes --max-rss when no explicit cap is given', async () => { const outFile = join(tmpdir(), `gbrain-sup-maxrss-auto-${process.pid}-${Date.now()}.txt`); try { unlinkSync(outFile); } catch { /* may not exist */ } const { resolveDefaultMaxRssMb } = await import('../src/core/minions/rss-default.ts'); const expected = resolveDefaultMaxRssMb(); const h = makeHarness('maxrss-auto', `printf '%s\\n' "$*" > "$OUT_FILE" ; exit 1`); try { const sup = spawnSupervisor(h, { OUT_FILE: outFile, SUP_MAX_CRASHES: '1', }); await sup.exited; expect(existsSync(outFile)).toBe(true); const argv = readFileSync(outFile, 'utf8').trim(); expect(argv).toContain(`--max-rss ${expected}`); // Auto-sized value is clamped into the sane range, never the old 2048 // unless the box genuinely resolves there. expect(expected).toBeGreaterThanOrEqual(4096); expect(expected).toBeLessThanOrEqual(16384); } finally { try { unlinkSync(outFile); } catch { /* noop */ } h.cleanup(); } }, 15_000); }); describe('integration: audit file rotation + helper', () => { it('computeSupervisorAuditFilename returns supervisor-YYYY-Www.jsonl format', () => { const jan15_2026 = new Date(Date.UTC(2026, 0, 15)); // Thu expect(computeSupervisorAuditFilename(jan15_2026)).toMatch(/^supervisor-2026-W\d\d\.jsonl$/); }); it('year-boundary ISO week: 2027-01-01 reports as 2026-W53', () => { const jan1_2027 = new Date(Date.UTC(2027, 0, 1)); // ISO week: 2027-01-01 is Friday of W53 of 2026 expect(computeSupervisorAuditFilename(jan1_2027)).toBe('supervisor-2026-W53.jsonl'); }); }); });