Files
gbrain/test/exit-classification.test.ts
T
0620094121 v0.35.5.1 fix(doctor): stop counting clean supervisor exits as crashes (#1108)
* feat(supervisor-audit): shared isCrashExit + summarizeCrashes classifier

Adds the read-side foundation for reading `likely_cause` off `worker_exited`
audit events. Denylist semantics — only `clean_exit` and `graceful_shutdown`
are non-crashes. Future unrecognized causes surface by default.

`isCrashExit(event)` classifies a single audit event with legacy
`code !== 0` fallback for pre-v0.34 entries lacking `likely_cause`.

`summarizeCrashes(events)` aggregates a 24h window into a `CrashSummary`
with per-cause counts (runtime_error, oom_or_external_kill, unknown,
legacy) and a `clean_exits` total.

Both helpers live next to `readSupervisorEvents` so the producer (the
JSONL writer) and the consumers (doctor + jobs CLI) share one regression
point. Test matrix pins all 9 isCrashExit branches plus 5 summarizeCrashes
aggregation cases including the future-cause denylist regression guard.

* fix(doctor,jobs): wire supervisor check to summarizeCrashes

`gbrain doctor` and `gbrain jobs supervisor status` both counted every
`worker_exited` audit event as a crash, regardless of `likely_cause`.
After v0.34.3.0 added RSS-watchdog drains (code=0), the count inflated
to 120+/day on a healthy brain — the alarm pattern users reported.

Both surfaces now go through `summarizeCrashes(events)` (single
regression point, can't drift). The warn threshold drops from `>3`
to `>=1` now that the counter is calibrated; the per-cause breakdown
(runtime=N oom=M unknown=K legacy=L) gives operators triage context
in the message without grep'ing the JSONL audit.

`gbrain jobs supervisor status --json` adds `crashes_by_cause` and
`clean_exits_24h` fields so monitoring dashboards bind to the named
buckets.

4 source-grep wiring assertions in doctor.test.ts pin both call sites
against drift.

* chore: bump version and changelog (v0.35.5.0)

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

* docs: document v0.35.5.0 supervisor-audit crash classifier

Add CLAUDE.md entry for src/core/minions/handlers/supervisor-audit.ts
covering the new isCrashExit/summarizeCrashes/CrashSummary/CLEAN_EXIT_CAUSES
exports. Extend doctor.ts and jobs.ts entries with the v0.35.5.0
wire-up: shared helper, denylist semantics, >=1 warn threshold, per-cause
breakdown in messages, crashes_by_cause + clean_exits_24h in JSON.
Regenerate llms-full.txt to match.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
2026-05-17 14:27:31 -07:00

123 lines
5.3 KiB
TypeScript

import { describe, it, expect } from 'bun:test';
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import { classifyWorkerExit } from '../src/core/minions/exit-classification.ts';
describe('classifyWorkerExit', () => {
it('code=0 → clean_exit', () => {
expect(classifyWorkerExit({ code: 0 })).toBe('clean_exit');
});
it('code=1 (runtime error) → crash', () => {
expect(classifyWorkerExit({ code: 1 })).toBe('crash');
});
it('code=137 (SIGKILL) → crash', () => {
expect(classifyWorkerExit({ code: 137 })).toBe('crash');
});
it('code=null (signal-only exit, audit JSON shape) → crash', () => {
expect(classifyWorkerExit({ code: null })).toBe('crash');
});
it('missing code (older audit shape with undefined key) → crash', () => {
// JSON.stringify drops undefined keys; reader may see {} or {code: null}.
// Both must classify as crash so a corrupted/legacy row doesn't get
// silently demoted into the clean-restart bucket.
expect(classifyWorkerExit({})).toBe('crash');
});
});
describe('consumer wire-up — helper used by all 3 sites (no inline filters left)', () => {
// The whole point of T7 (DRY) is replacing three inline filters with one
// helper. These tests pin the wire-up so a future refactor that
// accidentally inlines the rule again gets caught at test time, not at
// production-divergence time.
// v0.35.5.0: doctor.ts and jobs.ts moved from `classifyWorkerExit` (binary
// code-based) to `summarizeCrashes` (per-cause via `likely_cause`). The
// wire-up contract is "must use a shared helper, not an inline filter" —
// the specific helper differs by site. The supervisor's internal restart
// policy still uses `classifyWorkerExit` (binary is the right shape there).
const SITES = [
{ label: 'doctor.ts', path: 'src/commands/doctor.ts', helper: 'summarizeCrashes' },
{ label: 'jobs.ts', path: 'src/commands/jobs.ts', helper: 'summarizeCrashes' },
{ label: 'child-worker-supervisor.ts', path: 'src/core/minions/child-worker-supervisor.ts', helper: 'classifyWorkerExit' },
];
for (const site of SITES) {
it(`${site.label} uses a shared classifier helper (${site.helper})`, () => {
const source = readFileSync(join(import.meta.dir, '..', site.path), 'utf8');
// Helper is either imported by name (top-level) or via dynamic import.
const helperRe = new RegExp(`\\b${site.helper}\\b`);
expect(source).toMatch(helperRe);
});
it(`${site.label} calls ${site.helper} at least once`, () => {
const source = readFileSync(join(import.meta.dir, '..', site.path), 'utf8');
const callRe = new RegExp(`\\b${site.helper}\\s*\\(`);
expect(source).toMatch(callRe);
});
}
it('doctor.ts no longer has the inline `code !== 0 && code !== undefined` filter', () => {
const source = readFileSync(join(import.meta.dir, '..', 'src/commands/doctor.ts'), 'utf8');
// The pre-T7 inline filter; if this regex matches, the refactor leaked back.
expect(source).not.toMatch(/code !== 0\s*&&\s*\(?\s*\w+\s+as\s+any\s*\)?\.\s*code !== undefined/);
});
it('jobs.ts no longer has the inline filter', () => {
const source = readFileSync(join(import.meta.dir, '..', 'src/commands/jobs.ts'), 'utf8');
expect(source).not.toMatch(/code !== 0\s*&&\s*\(?\s*\w+\s+as\s+any\s*\)?\.\s*code !== undefined/);
});
it('child-worker-supervisor.ts uses helper to decide clean_exit vs crash branch', () => {
const source = readFileSync(join(import.meta.dir, '..', 'src/core/minions/child-worker-supervisor.ts'), 'utf8');
// The exit-handler branch should compare the helper result, not the raw code.
expect(source).toMatch(/classifyWorkerExit\(\s*\{\s*code\s*\}\s*\)\s*===\s*['"]clean_exit['"]/);
});
});
describe('audit-log shape integration — `code: 0` event is NOT counted as a crash', () => {
// Sanity round-trip: simulate the exact event shape that supervisor-audit
// writes (and that doctor + jobs read), classify it through the helper,
// and confirm the result. This catches future changes to the audit event
// shape (e.g. renaming `code` to `exit_code`) that would silently break
// the consumers' crash counting.
it('audit event { event: "worker_exited", code: 0, signal: null } → clean_exit', () => {
const auditEvent = {
event: 'worker_exited',
ts: '2026-05-15T12:00:00Z',
code: 0,
signal: null,
runDurationMs: 30000,
likelyCause: 'clean_exit',
};
expect(classifyWorkerExit(auditEvent as { code?: number | null })).toBe('clean_exit');
});
it('audit event { event: "worker_exited", code: 1, signal: null } → crash', () => {
const auditEvent = {
event: 'worker_exited',
ts: '2026-05-15T12:00:00Z',
code: 1,
signal: null,
runDurationMs: 250,
likelyCause: 'runtime_error',
};
expect(classifyWorkerExit(auditEvent as { code?: number | null })).toBe('crash');
});
it('audit event { event: "worker_exited", code: null, signal: "SIGKILL" } → crash', () => {
const auditEvent = {
event: 'worker_exited',
ts: '2026-05-15T12:00:00Z',
code: null,
signal: 'SIGKILL',
runDurationMs: 0,
likelyCause: 'oom_or_external_kill',
};
expect(classifyWorkerExit(auditEvent as { code?: number | null })).toBe('crash');
});
});