/** * Unit tests for the shared crash classifier used by `gbrain doctor` and * `gbrain jobs supervisor status`. Both surfaces import `isCrashExit` + * `summarizeCrashes` from `src/core/minions/handlers/supervisor-audit.ts`; * pinning them here keeps the two CLI surfaces from drifting. * * Why this file exists: pre-fix the doctor counted every `worker_exited` * event as a crash, regardless of `likely_cause`. Clean SIGTERM shutdowns * and RSS-watchdog drains (code=0) inflated `crashes_24h` to 120+/day on * Garry's brain. The classifier upstream in child-worker-supervisor.ts * already stamped `likely_cause` correctly; the read sites just ignored it. * These tests pin every branch of the new shared classifier so the bug * cannot silently recur. */ import { describe, test, expect } from 'bun:test'; import { isCrashExit, summarizeCrashes, type CrashSummary, } from '../src/core/minions/handlers/supervisor-audit.ts'; import type { SupervisorEmission } from '../src/core/minions/supervisor.ts'; // Helper: build a SupervisorEmission of the given event with arbitrary extra // fields. `ts` is required by the type but irrelevant to the classifier; we // stamp a constant so failures show predictable fixtures. function evt( event: SupervisorEmission['event'], extra: Record = {}, ): SupervisorEmission { return { event, ts: '2026-05-16T00:00:00Z', ...extra }; } describe('isCrashExit — branch matrix', () => { // Case 1: explicit clean exit (worker returned code=0 voluntarily, // e.g. RSS watchdog drain). The most common cause of the original // bug — every drain was being counted as a crash. test('clean_exit is not a crash', () => { expect(isCrashExit(evt('worker_exited', { likely_cause: 'clean_exit', code: 0 }))).toBe(false); }); // Case 2: SIGTERM-driven shutdown (operator stop, OS-initiated graceful // termination). Also not a crash. test('graceful_shutdown is not a crash', () => { expect(isCrashExit(evt('worker_exited', { likely_cause: 'graceful_shutdown', signal: 'SIGTERM' }))).toBe(false); }); // Case 3: code=1 from the worker process — a real runtime error. test('runtime_error is a crash', () => { expect(isCrashExit(evt('worker_exited', { likely_cause: 'runtime_error', code: 1 }))).toBe(true); }); // Case 4: SIGKILL (kernel OOM kill, external `kill -9`). Real crash. test('oom_or_external_kill is a crash', () => { expect(isCrashExit(evt('worker_exited', { likely_cause: 'oom_or_external_kill', signal: 'SIGKILL' }))).toBe(true); }); // Case 5: catch-all bucket from the upstream classifier (unusual code or // signal combination). Real crash. test('unknown is a crash', () => { expect(isCrashExit(evt('worker_exited', { likely_cause: 'unknown', code: 137 }))).toBe(true); }); // Case 6: denylist regression guard. If a future maintainer adds a NEW // `likely_cause` value upstream (e.g. `lock_lost`, `panic`, // `db_connection_lost`), the doctor MUST surface it by default. Allowlist // semantics would have silently misclassified this as clean — the exact // bug class this fix exists to close. test('unrecognized future likely_cause is a crash (denylist regression guard)', () => { expect(isCrashExit(evt('worker_exited', { likely_cause: 'future_value_not_known' }))).toBe(true); }); // Case 7: legacy fallback path — pre-v0.34 audit lines lacking // `likely_cause`. Use `code` to classify: code=0 is clean. test('legacy (no likely_cause) with code=0 is not a crash', () => { expect(isCrashExit(evt('worker_exited', { code: 0 }))).toBe(false); }); // Case 8: legacy fallback path — pre-v0.34 entry with code=1 (real crash). test('legacy (no likely_cause) with code=1 is a crash', () => { expect(isCrashExit(evt('worker_exited', { code: 1 }))).toBe(true); }); // Case 9: defensive — non-exit lifecycle events MUST never be counted as // crashes regardless of their other fields. Catches a future caller that // forgets the upstream `event === 'worker_exited'` filter. test('non-exit event is never a crash', () => { expect(isCrashExit(evt('started'))).toBe(false); expect(isCrashExit(evt('worker_spawned', { code: 1 }))).toBe(false); expect(isCrashExit(evt('max_crashes_exceeded', { likely_cause: 'runtime_error' }))).toBe(false); }); }); describe('summarizeCrashes — aggregation', () => { // Feed a representative mixed stream and assert every bucket. The mix // exercises every classifier branch so the message-format consumers // (doctor.ts and jobs.ts) get a stable shape. test('aggregates a mixed stream into per-cause buckets and clean_exits', () => { const events: SupervisorEmission[] = [ evt('worker_exited', { likely_cause: 'runtime_error', code: 1 }), evt('worker_exited', { likely_cause: 'runtime_error', code: 1 }), evt('worker_exited', { likely_cause: 'oom_or_external_kill', signal: 'SIGKILL' }), evt('worker_exited', { likely_cause: 'unknown', code: 137 }), evt('worker_exited', { code: 1 }), // legacy (no likely_cause) evt('worker_exited', { likely_cause: 'clean_exit', code: 0 }), evt('worker_exited', { likely_cause: 'clean_exit', code: 0 }), evt('worker_exited', { likely_cause: 'clean_exit', code: 0 }), evt('worker_exited', { likely_cause: 'graceful_shutdown', signal: 'SIGTERM' }), // Non-exit events MUST be ignored (no double-counting against either bucket). evt('started'), evt('worker_spawned'), evt('health_warn'), ]; const summary: CrashSummary = summarizeCrashes(events); expect(summary.total).toBe(5); expect(summary.by_cause.runtime_error).toBe(2); expect(summary.by_cause.oom_or_external_kill).toBe(1); expect(summary.by_cause.unknown).toBe(1); expect(summary.by_cause.legacy).toBe(1); expect(summary.clean_exits).toBe(4); // total + clean_exits should equal the count of worker_exited events, // proving non-exit lifecycle events were excluded from both buckets. const exitCount = events.filter((e) => e.event === 'worker_exited').length; expect(summary.total + summary.clean_exits).toBe(exitCount); }); test('empty input returns zero summary', () => { const summary = summarizeCrashes([]); expect(summary).toEqual({ total: 0, by_cause: { runtime_error: 0, oom_or_external_kill: 0, rss_watchdog: 0, unknown: 0, legacy: 0 }, clean_exits: 0, }); }); // issue #1678: rss_watchdog is a crash-classified cause (NOT in // CLEAN_EXIT_CAUSES) with its OWN bucket — operators watching // by_cause.rss_watchdog rise know the cap is too low for the workload, a // distinct signal from a generic runtime_error or OOM-killer SIGKILL. test('rss_watchdog routes to its own bucket, not legacy', () => { const summary = summarizeCrashes([ evt('worker_exited', { likely_cause: 'rss_watchdog' }), evt('worker_exited', { likely_cause: 'rss_watchdog' }), evt('worker_exited', { likely_cause: 'runtime_error' }), ]); expect(summary.total).toBe(3); expect(summary.by_cause.rss_watchdog).toBe(2); expect(summary.by_cause.runtime_error).toBe(1); expect(summary.by_cause.legacy).toBe(0); expect(summary.clean_exits).toBe(0); }); // isCrashExit treats rss_watchdog as a crash (it's a real problem), NOT a // clean exit — pins that the worker draining itself on a too-low cap shows // up in operator health surfaces instead of looking like a clean drain. test('isCrashExit classifies rss_watchdog as a crash', () => { expect(isCrashExit(evt('worker_exited', { likely_cause: 'rss_watchdog' }))).toBe(true); }); test('only non-exit events returns zero summary', () => { const summary = summarizeCrashes([evt('started'), evt('worker_spawned'), evt('stopped')]); expect(summary.total).toBe(0); expect(summary.clean_exits).toBe(0); }); // Denylist regression guard at the AGGREGATOR level. isCrashExit Case 6 // proves an unrecognized future `likely_cause` is counted as a crash; this // pins which BUCKET it lands in. The `else` branch in summarizeCrashes // routes any crash-classified event whose cause doesn't match the three // explicit buckets into `legacy`. Operators watching `legacy=N` rise know // the upstream classifier added a value the doctor doesn't yet name — // that's the intended signal. test('unrecognized future likely_cause routes to legacy bucket', () => { const summary = summarizeCrashes([ evt('worker_exited', { likely_cause: 'lock_lost' }), evt('worker_exited', { likely_cause: 'panic' }), ]); expect(summary.total).toBe(2); expect(summary.by_cause.legacy).toBe(2); expect(summary.by_cause.runtime_error).toBe(0); expect(summary.by_cause.oom_or_external_kill).toBe(0); expect(summary.by_cause.unknown).toBe(0); }); // Truly malformed legacy line — `likely_cause` missing AND `code` null // (or undefined). The classifier comment explicitly says "fail-loud, the // user can investigate the audit file directly", which means count it. // null !== 0 is true so isCrashExit returns true; summarizeCrashes then // lands it in legacy. Pinning this prevents a regression where a future // refactor adds `code != null` and silently drops malformed entries. test('legacy entry with null code counts as crash in legacy bucket', () => { const summary = summarizeCrashes([ evt('worker_exited', { code: null }), ]); expect(summary.total).toBe(1); expect(summary.by_cause.legacy).toBe(1); expect(summary.clean_exits).toBe(0); }); }); // issue #2227 bug #1/#2: a duplicate supervisor (different $HOME / --pid-file) // passes the pidfile guard but loses the queue-scoped DB singleton lock // (#1849) and `process.exit(ExitCodes.LOCK_HELD)`s. That fence-exit happens in // supervisor.ts:start() BEFORE the worker is ever spawned and BEFORE the // `started` event is emitted (the DB-lock acquire at :528 precedes the // `started` emit at :555), so the loser contributes NO `worker_exited` events // to the audit trail. The crash ledger (summarizeCrashes) moves ONLY on // `worker_exited`, so a fence-collision can never burn the crash budget the // guard exists to protect. The field report claimed 20 "unknown" crashes were // fence exits; this pins that the fence path is structurally uncountable, so a // future refactor that (wrongly) logs a `worker_exited` on the LOCK_HELD path // would fail here instead of silently re-introducing the breaker-trip loop. describe('summarizeCrashes — LOCK_HELD fence-exit is never a crash (issue #2227)', () => { test('a losing duplicate supervisor (no worker_exited) contributes zero crashes', () => { // The loser exits at the DB-lock fence before emitting `started`, so its // audit contribution is empty. Even paired with the winner's healthy // stream, total crashes stay 0. const winnerHealthy: SupervisorEmission[] = [ evt('started', { supervisor_pid: 4242, queue: 'default', concurrency: 3 }), evt('worker_spawned', { worker_pid: 4243 }), // ... no worker_exited yet (winner still running) ]; expect(summarizeCrashes(winnerHealthy).total).toBe(0); }); test('started + stopped with no worker_exited (clean fence path) is zero crashes', () => { // Defensive: even if a future change made the loser emit lifecycle events // (started/shutting_down/stopped) around the LOCK_HELD exit, none of those // are `worker_exited`, so the ledger must stay at 0. The only way this test // fails is if someone logs a `worker_exited` on the fence path — which is // exactly the regression #2227 fix #2 guards against. const fenceStream: SupervisorEmission[] = [ evt('started', { supervisor_pid: 5252 }), evt('shutting_down', { reason: 'LOCK_HELD' }), evt('stopped', { exit_code: 2 }), ]; const summary = summarizeCrashes(fenceStream); expect(summary.total).toBe(0); expect(summary.clean_exits).toBe(0); }); });