From 60b3e099aebe81073b9c5e73fd277807cbb7d29d Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Tue, 2 Jun 2026 21:28:13 -0700 Subject: [PATCH] feat(doctor): worker_oom_loop + pool_reap_health checks + cause-ranked top_issues (#1685 GAP A/B/C) - computeWorkerOomLoopCheck: unions supervisor rss_watchdog + minion_jobs watchdog-abort (CODEX #5), cap fallback to resolveDefaultMaxRssMb (CODEX #6) - computePoolReapHealthCheck: reaps-not-recovering fail, thrash warn - doctor-cause-rank rankIssues: tier ordering + grounded downstream_of (CODEX #9) + drift guard (4A) - supervisor causeStr + queue_health cross-reference worker_oom_loop (DRY 1C) - register both checks in doctor-categories ops --- src/commands/doctor.ts | 217 ++++++++++++++++++++++++++- src/core/doctor-categories.ts | 2 + src/core/doctor-cause-rank.ts | 149 ++++++++++++++++++ test/doctor-cause-rank.test.ts | 87 +++++++++++ test/doctor-pool-reap-health.test.ts | 64 ++++++++ test/doctor-worker-oom-loop.test.ts | 101 +++++++++++++ 6 files changed, 617 insertions(+), 3 deletions(-) create mode 100644 src/core/doctor-cause-rank.ts create mode 100644 test/doctor-cause-rank.test.ts create mode 100644 test/doctor-pool-reap-health.test.ts create mode 100644 test/doctor-worker-oom-loop.test.ts diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index ed665a7dd..3a3e42db0 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -21,6 +21,7 @@ import { loadCompletedMigrations } from '../core/preferences.ts'; import { compareVersions } from './migrations/index.ts'; import { createProgress, startHeartbeat, type ProgressReporter } from '../core/progress.ts'; import { categorizeCheck, type CheckCategory } from '../core/doctor-categories.ts'; +import { rankIssues, type RankedIssue } from '../core/doctor-cause-rank.ts'; import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts'; import type { DbUrlSource } from '../core/config.ts'; import { gbrainPath } from '../core/config.ts'; @@ -113,6 +114,12 @@ export interface DoctorReport { meta: number; }; checks: Check[]; + /** + * v0.42.x (#1685 GAP C) — non-ok checks ranked by cause (root before symptom, + * fail before warn). Lets an agent act on the root cause without re-deriving + * the ranking. Additive + optional; schema_version stays at 2. + */ + top_issues?: RankedIssue[]; } function _penaltyScore(checks: Check[]): number { @@ -165,6 +172,7 @@ export function computeDoctorReport(checks: Check[]): DoctorReport { meta: _penaltyScore(meta), }, checks: tagged, + top_issues: rankIssues(tagged), }; } @@ -3167,6 +3175,170 @@ export async function checkCycleFreshness( * - `progress` reporter writes to stderr (heartbeats per check) * - `engine.executeRaw` / handler-leaf calls (the actual probe work) */ +/** + * issue #1685 (GAP A) — the single authoritative "worker is OOM-looping" signal. + * + * One `gbrain doctor` line replaces the hours of log archaeology the #1678 + * incident required: `cap=8192MB, N watchdog kills/24h → raise --max-rss`. + * + * UNIONS two sources so it's authoritative for BOTH worker modes (CODEX #5): + * - SUPERVISED workers: supervisor audit `worker_exited likely_cause=rss_watchdog`, + * read cross-week (CODEX #7) so a Mon read doesn't lose a Sun loop. + * - BARE `gbrain jobs work`: NO supervisor event is written; the only trace is + * `minion_jobs.error_text = 'aborted: watchdog'` (the same source queue_health + * subcheck 3 reads). Reading supervisor-only would miss bare workers entirely + * and the queue_health cross-reference would point at an unemitted check. + * + * Cap (CODEX #6): the breaker alert stamps `max_rss_mb`, but a fail from + * oomKills>=5 spread over 24h may have no breaker event → no stamped cap. Fall + * back to `resolveDefaultMaxRssMb()` so the message always renders a number. + * + * Returns null when the worker never OOM'd (don't warn installs that never hit + * it). Pure-ish: filesystem audit read + one minion_jobs count; no process.exit. + * Exported so `test/doctor-worker-oom-loop.test.ts` drives it directly. + */ +export async function computeWorkerOomLoopCheck( + engine: BrainEngine | null, +): Promise { + let supervisorKills = 0; + let capFromBreaker: number | null = null; + let breakerTripped = false; + try { + const { readRecentSupervisorEvents, summarizeCrashes } = await import( + '../core/minions/handlers/supervisor-audit.ts' + ); + const events = readRecentSupervisorEvents(24); + supervisorKills = summarizeCrashes(events).by_cause.rss_watchdog; + // Latest rss_watchdog_loop breaker alert carries the cap the supervisor + // spawned with (supervisor.ts:521); its presence also means the breaker + // tripped. Walk all events; last one wins for the cap. + for (const e of events) { + const row = e as Record; + if (e.event === 'health_warn' && row.reason === 'rss_watchdog_loop') { + breakerTripped = true; + const cap = Number(row.max_rss_mb); + if (Number.isFinite(cap) && cap > 0) capFromBreaker = cap; + } + } + } catch { + // supervisor-audit read is best-effort; fall through to minion_jobs. + } + + let bareWorkerKills = 0; + if (engine && engine.kind !== 'pglite') { + try { + const sql = db.getConnection(); + const rows: Array<{ cnt: number }> = await sql` + SELECT count(*)::int AS cnt + FROM minion_jobs + WHERE status IN ('dead', 'failed') + AND finished_at > now() - interval '24 hours' + AND error_text = 'aborted: watchdog' + `; + bareWorkerKills = rows[0]?.cnt ?? 0; + } catch { + // minion_jobs may not exist on a fresh brain; best-effort. + } + } + + // De-dup note (CODEX #5 accepted trade-off): a supervised watchdog kill aborts + // in-flight jobs, so it can show in BOTH counts. We accept slight over-count + // rather than miss bare workers — the signal is "is it OOM-looping," not an + // exact tally. `details` keeps the two sources separate for honesty. + const oomKills = supervisorKills + bareWorkerKills; + if (oomKills < 1 && !breakerTripped) return null; + + let capMb: number; + let capSource: 'breaker' | 'default'; + if (capFromBreaker !== null) { + capMb = capFromBreaker; + capSource = 'breaker'; + } else { + let def = 16384; + try { + const { resolveDefaultMaxRssMb } = await import('../core/minions/rss-default.ts'); + def = resolveDefaultMaxRssMb(); + } catch { + // keep the conservative ceiling fallback. + } + capMb = def; + capSource = 'default'; + } + + const fixHint = + 'raise --max-rss (gbrain jobs work --max-rss ; auto-sizes to min(0.5×RAM,16GB))'; + const capLabel = capSource === 'breaker' ? `cap=${capMb}MB` : `cap≈${capMb}MB (auto-sized default)`; + const status: Check['status'] = breakerTripped || oomKills >= 5 ? 'fail' : 'warn'; + return { + name: 'worker_oom_loop', + status, + message: + `Worker OOM-looping: ${capLabel}, ${oomKills} watchdog kill(s)/24h → ${fixHint}. ` + + `Peak RSS: see worker stderr.`, + details: { + oom_kills: oomKills, + supervisor_kills: supervisorKills, + bare_worker_kills: bareWorkerKills, + cap_mb: capMb, + cap_source: capSource, + breaker_tripped: breakerTripped, + fix_hint: fixHint, + }, + }; +} + +/** + * issue #1685 (GAP B) — DB pool reap health (Postgres-only). + * + * Answers the #1685 line "DB pool reaped N times/hr AND not auto-recovering" + * that no existing signal expresses. Reads the pool-recovery audit + * (`reconnect()` emits reap_detected / reconnect_succeeded / reconnect_failed): + * - fail: reaps>0 AND reconnect failures>0 → the pool is being reaped and + * rebuilds are throwing (genuinely not recovering). + * - warn: reaps>=10/hr, all recovered → pooler thrash (self-heal works but the + * cap is likely too low / concurrency too high). + * - else: null (quiet — a few reaps that all recovered is normal). + * + * Returns null on PGLite / no engine / audit-read failure. Exported so + * `test/doctor-pool-reap-health.test.ts` drives it directly. + */ +export async function computePoolReapHealthCheck( + engine: BrainEngine | null, +): Promise { + if (!engine || engine.kind === 'pglite') return null; + let r: { reaps: number; recoveries: number; failures: number }; + try { + const { readRecentPoolRecoveries } = await import('../core/audit/pool-recovery-audit.ts'); + r = readRecentPoolRecoveries(1); + } catch { + return null; + } + + if (r.reaps > 0 && r.failures > 0) { + const fix = 'check DB reachability / credentials (reconnect is throwing)'; + return { + name: 'pool_reap_health', + status: 'fail', + message: + `DB pool reaped ${r.reaps}× and reconnect FAILED ${r.failures}× in last hour ` + + `— not auto-recovering; ${fix}.`, + details: { reaps: r.reaps, recoveries: r.recoveries, failures: r.failures, fix_hint: fix }, + }; + } + if (r.reaps >= 10) { + const fix = 'raise --max-rss or reduce worker concurrency (pooler thrash)'; + return { + name: 'pool_reap_health', + status: 'warn', + message: + `DB pool reaped ${r.reaps}× in last hour (self-heal recovered each) ` + + `— ${fix}.`, + details: { reaps: r.reaps, recoveries: r.recoveries, failures: r.failures, fix_hint: fix }, + }; + } + return null; +} + export async function buildChecks( engine: BrainEngine | null, args: string[], @@ -3446,7 +3618,7 @@ export async function buildChecks( // shape is the right contract. const summary = summarizeCrashes(events); const crashes24h = summary.total; - const causeStr = `runtime=${summary.by_cause.runtime_error} oom=${summary.by_cause.oom_or_external_kill} unknown=${summary.by_cause.unknown} legacy=${summary.by_cause.legacy}`; + const causeStr = `runtime=${summary.by_cause.runtime_error} oom=${summary.by_cause.oom_or_external_kill} rss=${summary.by_cause.rss_watchdog} unknown=${summary.by_cause.unknown} legacy=${summary.by_cause.legacy}${summary.by_cause.rss_watchdog > 0 ? ' (see worker_oom_loop)' : ''}`; const maxCrashesEvent = events.filter(e => e.event === 'max_crashes_exceeded').pop() ?? null; // Only surface a Check if the supervisor was ever observed (stops the @@ -3485,6 +3657,27 @@ export async function buildChecks( // Audit read / import failure is best-effort; skip silently. } + // 3b-quater. Worker OOM-loop (issue #1685 GAP A) — the single authoritative + // "is the worker OOM-looping" line, unioning supervised (supervisor audit) + // and bare-worker (minion_jobs watchdog-abort) kills. Returns null when the + // worker never OOM'd, so clean installs see nothing. + try { + const oomCheck = await computeWorkerOomLoopCheck(engine); + if (oomCheck) checks.push(oomCheck); + } catch { + // best-effort. + } + + // 3b-quinquies. DB pool reap health (issue #1685 GAP B) — Postgres pooler + // reap frequency + recovered-vs-stuck split. Quiet unless reaps thrash or + // reconnect is failing. + try { + const reapCheck = await computePoolReapHealthCheck(engine); + if (reapCheck) checks.push(reapCheck); + } catch { + // best-effort. + } + // 3b-tris. Stub-guard fire count (last 24h). The v0.34.5 stub guard in // fence-write.ts refuses to spawn unprefixed entity pages (e.g. bare // `alice.md` at brain root). Each fire is appended to @@ -5762,9 +5955,8 @@ export async function buildChecks( if (rssKillCount > 0) { problems.push( `${rssKillCount} job(s) dead-lettered for RSS-watchdog memory-limit kills in last 24h. ` + - `v0.22.14 changed the bare-worker --max-rss default from 0 (off) to 2048 MB. ` + `Fix: raise the limit (e.g. \`gbrain jobs work --max-rss 4096\`) or opt out (\`--max-rss 0\`). ` + - `See skills/migrations/v0.22.14.md.` + `→ see worker_oom_loop for the cap + fix (the authoritative OOM-loop signal).` ); } if (promptTooLongCount > 0) { @@ -6327,6 +6519,25 @@ function outputResults(checks: Check[], json: boolean): boolean { console.log('\nGBrain Health Check'); console.log('==================='); + + // #1685 GAP C — cause-ranked summary so the operator reads the root cause + // first instead of scrolling the full list. Caps at 5; clean brains skip it. + const topIssues = report.top_issues ?? []; + if (topIssues.length > 0) { + console.log(''); + console.log('Top issues (ranked by cause):'); + const shown = topIssues.slice(0, 5); + for (const issue of shown) { + const icon = issue.status === 'fail' ? 'FAIL' : 'WARN'; + const dn = issue.downstream_of ? ` (likely downstream of ${issue.downstream_of})` : ''; + console.log(` [${icon}] ${issue.name}${dn} → ${issue.fix}`); + } + if (topIssues.length > shown.length) { + console.log(` +${topIssues.length - shown.length} more — see full list below`); + } + console.log(''); + } + for (const c of report.checks) { const icon = c.status === 'ok' ? 'OK' : c.status === 'warn' ? 'WARN' : 'FAIL'; console.log(` [${icon}] ${c.name}: ${c.message}`); diff --git a/src/core/doctor-categories.ts b/src/core/doctor-categories.ts index 680794e21..ce9323a1b 100644 --- a/src/core/doctor-categories.ts +++ b/src/core/doctor-categories.ts @@ -138,11 +138,13 @@ export const OPS_CHECK_NAMES: ReadonlySet = new Set([ 'rls', 'rls_event_trigger', 'search_mode', + 'pool_reap_health', 'stale_locks', 'subagent_capability', 'subagent_health', 'supervisor', 'sync_consolidation', + 'worker_oom_loop', 'ze_embedding_health', ]); diff --git a/src/core/doctor-cause-rank.ts b/src/core/doctor-cause-rank.ts new file mode 100644 index 000000000..647c6ba8b --- /dev/null +++ b/src/core/doctor-cause-rank.ts @@ -0,0 +1,149 @@ +/** + * issue #1685 (GAP C) — cause-ranked doctor issues. + * + * The #1685 posture ask: `gbrain doctor` is the single health truth, and it + * surfaces CAUSE before symptoms. During the #1678 incident the loud lines were + * all downstream DB-cascade noise (CONNECTION_ENDED, lock-renewal-failed) while + * the one true cause (RSS-watchdog OOM kill) scrolled by once. This module ranks + * the non-ok checks so the operator reads root causes first. + * + * HONESTY CONTRACT (CODEX #9): two checks both failing does NOT prove one caused + * the other. So: + * - Tier membership (root vs symptom) is ORDERING ONLY. It sorts roots above + * symptoms; it asserts NO causality. + * - `downstream_of` — the one place we DO claim a causal link — is set ONLY + * from a small map of KNOWN, grounded edges, AND only when the named root is + * itself in the failing set. It is deliberately NOT a root×symptom cartesian. + * "Everything failing is downstream of every root" is the false-precision we + * refuse to ship. + * + * Pure: no I/O, no engine, no process.exit. Unit-tested directly by + * `test/doctor-cause-rank.test.ts`, including the drift guard that every name in + * the cause graph still exists in `doctor-categories.ts` (DECISION 4A). + */ + +import { + BRAIN_CHECK_NAMES, + SKILL_CHECK_NAMES, + OPS_CHECK_NAMES, + META_CHECK_NAMES, +} from './doctor-categories.ts'; + +/** Minimal structural shape of a doctor Check that ranking needs. */ +export interface RankableCheck { + name: string; + status: 'ok' | 'warn' | 'fail'; + message: string; + details?: Record; +} + +export interface RankedIssue { + name: string; + status: 'warn' | 'fail'; + /** + * Coarse sort bucket. `root` = a designated root-cause check; `symptom` = + * everything else (NOT a proof that it's a downstream effect — just "not on + * the root-cause list"). The precise causal claim lives in `downstream_of`. + */ + tier: 'root' | 'symptom'; + /** + * Set ONLY for a known causal edge whose root is also failing. Absent + * otherwise — we never invent causality from co-occurrence. + */ + downstream_of?: string; + /** One-line fix. Prefers `details.fix_hint`; falls back to the check message. */ + fix: string; +} + +/** + * Checks that, when failing, are usually the DISEASE. Sorted to the top so the + * operator reads the cause first. Membership is ORDERING ONLY (CODEX #9). + */ +export const ROOT_CAUSE_CHECKS: ReadonlySet = new Set([ + 'worker_oom_loop', + 'pool_reap_health', + 'connection', + 'sync_freshness', + 'schema_version', +]); + +/** + * Checks that are commonly DOWNSTREAM noise during an incident. Sorted below + * roots. Ordering only — not a causal claim. + */ +export const SYMPTOM_CHECKS: ReadonlySet = new Set([ + 'queue_health', + 'batch_retry_health', + 'supervisor', + 'stale_locks', +]); + +/** + * KNOWN causal edges (symptom → root). `downstream_of` is set ONLY from this + * map AND ONLY when the named root is itself failing. Each edge is a real, + * grounded link, not a taxonomy guess: + * - queue_health → worker_oom_loop: an RSS-watchdog OOM kill aborts in-flight + * jobs; queue_health reads the SAME `error_text='aborted: watchdog'` rows + * that worker_oom_loop counts for bare workers. + * - supervisor → worker_oom_loop: the watchdog drain is a supervisor + * `worker_exited likely_cause=rss_watchdog` — the exact event + * worker_oom_loop's supervised half counts. + */ +const DOWNSTREAM_EDGES: Readonly> = { + queue_health: 'worker_oom_loop', + supervisor: 'worker_oom_loop', +}; + +function tierOf(name: string): 'root' | 'symptom' { + return ROOT_CAUSE_CHECKS.has(name) ? 'root' : 'symptom'; +} + +/** + * Rank non-ok checks: fail before warn, root before symptom, then name + * (deterministic). Returns the full ranked list; the renderer caps to top-N. + */ +export function rankIssues(checks: RankableCheck[]): RankedIssue[] { + const failing = checks.filter((c) => c.status !== 'ok'); + const failingNames = new Set(failing.map((c) => c.name)); + + const issues: RankedIssue[] = failing.map((c) => { + const root = DOWNSTREAM_EDGES[c.name]; + const downstream_of = root && failingNames.has(root) ? root : undefined; + const hint = c.details?.fix_hint; + const fix = + typeof hint === 'string' && hint.trim().length > 0 ? hint : c.message; + return { + name: c.name, + status: c.status as 'warn' | 'fail', + tier: tierOf(c.name), + ...(downstream_of ? { downstream_of } : {}), + fix, + }; + }); + + const statusRank = (s: string): number => (s === 'fail' ? 0 : 1); + const tierRank = (t: string): number => (t === 'root' ? 0 : 1); + issues.sort( + (a, b) => + statusRank(a.status) - statusRank(b.status) || + tierRank(a.tier) - tierRank(b.tier) || + a.name.localeCompare(b.name), + ); + return issues; +} + +/** Every name referenced by the cause graph (tiers). Drift-guard target (4A). */ +export const CAUSE_GRAPH_NAMES: ReadonlySet = new Set([ + ...ROOT_CAUSE_CHECKS, + ...SYMPTOM_CHECKS, +]); + +/** Union of all category-known check names — the drift-guard comparison set. */ +export function allKnownCheckNames(): ReadonlySet { + return new Set([ + ...BRAIN_CHECK_NAMES, + ...SKILL_CHECK_NAMES, + ...OPS_CHECK_NAMES, + ...META_CHECK_NAMES, + ]); +} diff --git a/test/doctor-cause-rank.test.ts b/test/doctor-cause-rank.test.ts new file mode 100644 index 000000000..18dc05dac --- /dev/null +++ b/test/doctor-cause-rank.test.ts @@ -0,0 +1,87 @@ +/** + * #1685 GAP C — cause-ranked doctor issues. Pure unit tests. + * + * Covers: fail-before-warn + root-before-symptom ordering, evidence-gated + * downstream_of (NEVER from co-occurrence alone — CODEX #9), fix-hint + * preference, and the DECISION 4A drift guard (every cause-graph name still + * exists in doctor-categories). + */ +import { describe, it, expect } from 'bun:test'; +import { + rankIssues, + ROOT_CAUSE_CHECKS, + SYMPTOM_CHECKS, + CAUSE_GRAPH_NAMES, + allKnownCheckNames, + type RankableCheck, +} from '../src/core/doctor-cause-rank.ts'; + +const ok = (name: string): RankableCheck => ({ name, status: 'ok', message: 'fine' }); +const warn = (name: string, msg = 'warned'): RankableCheck => ({ name, status: 'warn', message: msg }); +const fail = (name: string, msg = 'failed'): RankableCheck => ({ name, status: 'fail', message: msg }); + +describe('rankIssues', () => { + it('drops ok checks, returns [] when everything is healthy', () => { + expect(rankIssues([ok('connection'), ok('queue_health')])).toEqual([]); + }); + + it('orders fail before warn', () => { + const out = rankIssues([warn('stale_locks'), fail('schema_version')]); + expect(out.map((i) => i.name)).toEqual(['schema_version', 'stale_locks']); + }); + + it('orders root before symptom within the same status', () => { + // queue_health (symptom) + worker_oom_loop (root), both fail. + const out = rankIssues([fail('queue_health'), fail('worker_oom_loop')]); + expect(out.map((i) => i.name)).toEqual(['worker_oom_loop', 'queue_health']); + expect(out[0].tier).toBe('root'); + expect(out[1].tier).toBe('symptom'); + }); + + it('tags downstream_of ONLY when the named root is itself failing', () => { + const both = rankIssues([fail('worker_oom_loop'), fail('queue_health')]); + const q1 = both.find((i) => i.name === 'queue_health')!; + expect(q1.downstream_of).toBe('worker_oom_loop'); + }); + + it('does NOT tag downstream_of when the root is absent (no co-occurrence guess)', () => { + const out = rankIssues([fail('queue_health')]); // worker_oom_loop not failing + const q = out.find((i) => i.name === 'queue_health')!; + expect(q.downstream_of).toBeUndefined(); + }); + + it('does NOT tag downstream_of from a generic root×symptom cartesian', () => { + // schema_version is a root and stale_locks is a symptom, but there is NO + // declared causal edge between them — co-occurrence must not invent one. + const out = rankIssues([fail('schema_version'), fail('stale_locks')]); + const s = out.find((i) => i.name === 'stale_locks')!; + expect(s.downstream_of).toBeUndefined(); + }); + + it('uses details.fix_hint when present, else the message', () => { + const out = rankIssues([ + { name: 'worker_oom_loop', status: 'fail', message: 'long message', details: { fix_hint: 'raise --max-rss' } }, + warn('orphan_ratio', 'too many orphans'), + ]); + expect(out.find((i) => i.name === 'worker_oom_loop')!.fix).toBe('raise --max-rss'); + expect(out.find((i) => i.name === 'orphan_ratio')!.fix).toBe('too many orphans'); + }); + + it('is deterministic (name tiebreak) for same status+tier', () => { + const out = rankIssues([warn('zeta_unknown'), warn('alpha_unknown')]); + expect(out.map((i) => i.name)).toEqual(['alpha_unknown', 'zeta_unknown']); + }); +}); + +describe('DECISION 4A drift guard', () => { + it('every cause-graph name exists in doctor-categories known names', () => { + const known = allKnownCheckNames(); + const missing = [...CAUSE_GRAPH_NAMES].filter((n) => !known.has(n)); + expect(missing).toEqual([]); + }); + + it('root and symptom sets are disjoint', () => { + const overlap = [...ROOT_CAUSE_CHECKS].filter((n) => SYMPTOM_CHECKS.has(n)); + expect(overlap).toEqual([]); + }); +}); diff --git a/test/doctor-pool-reap-health.test.ts b/test/doctor-pool-reap-health.test.ts new file mode 100644 index 000000000..4956065c0 --- /dev/null +++ b/test/doctor-pool-reap-health.test.ts @@ -0,0 +1,64 @@ +// #1685 GAP B — pool_reap_health doctor check. +// +// computePoolReapHealthCheck only touches engine.kind + the pool-recovery audit +// (filesystem), so a minimal `{ kind: 'postgres' }` stub drives it hermetically. + +import { describe, expect, test, beforeEach, afterEach } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { withEnv } from './helpers/with-env.ts'; +import { logPoolRecovery } from '../src/core/audit/pool-recovery-audit.ts'; +import { computePoolReapHealthCheck } from '../src/commands/doctor.ts'; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const pg = { kind: 'postgres' } as any; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const pglite = { kind: 'pglite' } as any; + +let tmpDir: string; +beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pool-reap-health-')); +}); +afterEach(() => { + try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* best-effort */ } +}); + +describe('computePoolReapHealthCheck', () => { + test('null on PGLite (no pool) and on null engine', async () => { + expect(await computePoolReapHealthCheck(pglite)).toBeNull(); + expect(await computePoolReapHealthCheck(null)).toBeNull(); + }); + + test('fail when reaps>0 AND reconnect failed (not auto-recovering)', async () => { + await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => { + logPoolRecovery('reap_detected'); + logPoolRecovery('reconnect_failed', new Error('EHOSTUNREACH')); + const c = await computePoolReapHealthCheck(pg); + expect(c?.status).toBe('fail'); + expect(c?.message).toContain('not auto-recovering'); + expect(c?.name).toBe('pool_reap_health'); + }); + }); + + test('warn on pooler thrash (>=10 reaps all recovered)', async () => { + await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => { + for (let i = 0; i < 12; i++) { + logPoolRecovery('reap_detected'); + logPoolRecovery('reconnect_succeeded'); + } + const c = await computePoolReapHealthCheck(pg); + expect(c?.status).toBe('warn'); + expect(c?.message).toContain('12×'); + }); + }); + + test('null (quiet) when a few reaps all recovered', async () => { + await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => { + logPoolRecovery('reap_detected'); + logPoolRecovery('reconnect_succeeded'); + const c = await computePoolReapHealthCheck(pg); + expect(c).toBeNull(); + }); + }); +}); diff --git a/test/doctor-worker-oom-loop.test.ts b/test/doctor-worker-oom-loop.test.ts new file mode 100644 index 000000000..31754515d --- /dev/null +++ b/test/doctor-worker-oom-loop.test.ts @@ -0,0 +1,101 @@ +// #1685 GAP A — worker_oom_loop doctor check. +// +// Hermetic: writes synthetic supervisor audit JSONL into a GBRAIN_AUDIT_DIR +// tempdir and drives computeWorkerOomLoopCheck with engine=null (supervised +// half + cap logic + thresholds; the minion_jobs bare-worker branch is +// Postgres-only and mirrors the queue_health subcheck-3 query covered by E2E). +// Also pins the cross-week supervisor reader (CODEX #7). + +import { describe, expect, test, beforeEach, afterEach } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { withEnv } from './helpers/with-env.ts'; +import { computeWorkerOomLoopCheck } from '../src/commands/doctor.ts'; +import { + computeSupervisorAuditFilename, + readRecentSupervisorEvents, +} from '../src/core/minions/handlers/supervisor-audit.ts'; + +let tmpDir: string; +beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'worker-oom-loop-')); +}); +afterEach(() => { + try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* best-effort */ } +}); + +function writeSupervisorRows(rows: object[], fileDate = new Date()): void { + const file = path.join(tmpDir, computeSupervisorAuditFilename(fileDate)); + fs.writeFileSync(file, rows.map((r) => JSON.stringify(r)).join('\n') + '\n', 'utf8'); +} + +const nowIso = () => new Date().toISOString(); +const rssKill = () => ({ event: 'worker_exited', ts: nowIso(), likely_cause: 'rss_watchdog', code: 12 }); +const breaker = (cap: number) => ({ event: 'health_warn', ts: nowIso(), reason: 'rss_watchdog_loop', max_rss_mb: cap }); + +describe('computeWorkerOomLoopCheck', () => { + test('fail with breaker-stamped cap when the breaker tripped', async () => { + await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => { + writeSupervisorRows([rssKill(), rssKill(), rssKill(), rssKill(), rssKill(), rssKill(), breaker(2048)]); + const c = await computeWorkerOomLoopCheck(null); + expect(c?.status).toBe('fail'); + expect(c?.name).toBe('worker_oom_loop'); + expect(c?.message).toContain('cap=2048MB'); + expect(c?.message).toContain('raise --max-rss'); + expect(c?.details?.oom_kills).toBe(6); + expect(c?.details?.supervisor_kills).toBe(6); + expect(c?.details?.bare_worker_kills).toBe(0); + expect(c?.details?.cap_source).toBe('breaker'); + expect(c?.details?.breaker_tripped).toBe(true); + }); + }); + + test('warn with auto-sized cap fallback when no breaker event stamped a cap (CODEX #6)', async () => { + await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => { + writeSupervisorRows([rssKill(), rssKill()]); + const c = await computeWorkerOomLoopCheck(null); + expect(c?.status).toBe('warn'); + expect(c?.details?.cap_source).toBe('default'); + expect(c?.message).toContain('auto-sized default'); + expect(c?.details?.oom_kills).toBe(2); + }); + }); + + test('fail at >=5 kills even without a breaker event', async () => { + await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => { + writeSupervisorRows([rssKill(), rssKill(), rssKill(), rssKill(), rssKill()]); + const c = await computeWorkerOomLoopCheck(null); + expect(c?.status).toBe('fail'); + expect(c?.details?.breaker_tripped).toBe(false); + }); + }); + + test('null when the worker never OOM-looped', async () => { + await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => { + writeSupervisorRows([{ event: 'started', ts: nowIso() }]); + expect(await computeWorkerOomLoopCheck(null)).toBeNull(); + }); + }); + + test('null on an empty audit dir', async () => { + await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => { + expect(await computeWorkerOomLoopCheck(null)).toBeNull(); + }); + }); +}); + +describe('readRecentSupervisorEvents — cross-week (CODEX #7)', () => { + test('reads a within-24h event from the PREVIOUS ISO-week file', async () => { + await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => { + // Event timestamped within the 24h window but written into last week's + // file (the Monday-reads-Sunday case). The single-file reader would miss + // it; the cross-week reader must find it. + const prevWeekFileDate = new Date(Date.now() - 7 * 86400000); + writeSupervisorRows([rssKill()], prevWeekFileDate); + const events = readRecentSupervisorEvents(24); + expect(events.length).toBe(1); + expect(events[0].event).toBe('worker_exited'); + }); + }); +});