mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-31 04:07:52 +00:00
feat(doctor): worker_oom_loop + pool_reap_health checks + cause-ranked top_issues (#1685 GAP A/B/C)
- computeWorkerOomLoopCheck: unions supervisor rss_watchdog + minion_jobs watchdog-abort (CODEX #5), cap fallback to resolveDefaultMaxRssMb (CODEX #6) - computePoolReapHealthCheck: reaps-not-recovering fail, thrash warn - doctor-cause-rank rankIssues: tier ordering + grounded downstream_of (CODEX #9) + drift guard (4A) - supervisor causeStr + queue_health cross-reference worker_oom_loop (DRY 1C) - register both checks in doctor-categories ops
This commit is contained in:
+214
-3
@@ -21,6 +21,7 @@ import { loadCompletedMigrations } from '../core/preferences.ts';
|
||||
import { compareVersions } from './migrations/index.ts';
|
||||
import { createProgress, startHeartbeat, type ProgressReporter } from '../core/progress.ts';
|
||||
import { categorizeCheck, type CheckCategory } from '../core/doctor-categories.ts';
|
||||
import { rankIssues, type RankedIssue } from '../core/doctor-cause-rank.ts';
|
||||
import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
|
||||
import type { DbUrlSource } from '../core/config.ts';
|
||||
import { gbrainPath } from '../core/config.ts';
|
||||
@@ -113,6 +114,12 @@ export interface DoctorReport {
|
||||
meta: number;
|
||||
};
|
||||
checks: Check[];
|
||||
/**
|
||||
* v0.42.x (#1685 GAP C) — non-ok checks ranked by cause (root before symptom,
|
||||
* fail before warn). Lets an agent act on the root cause without re-deriving
|
||||
* the ranking. Additive + optional; schema_version stays at 2.
|
||||
*/
|
||||
top_issues?: RankedIssue[];
|
||||
}
|
||||
|
||||
function _penaltyScore(checks: Check[]): number {
|
||||
@@ -165,6 +172,7 @@ export function computeDoctorReport(checks: Check[]): DoctorReport {
|
||||
meta: _penaltyScore(meta),
|
||||
},
|
||||
checks: tagged,
|
||||
top_issues: rankIssues(tagged),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -3167,6 +3175,170 @@ export async function checkCycleFreshness(
|
||||
* - `progress` reporter writes to stderr (heartbeats per check)
|
||||
* - `engine.executeRaw` / handler-leaf calls (the actual probe work)
|
||||
*/
|
||||
/**
|
||||
* issue #1685 (GAP A) — the single authoritative "worker is OOM-looping" signal.
|
||||
*
|
||||
* One `gbrain doctor` line replaces the hours of log archaeology the #1678
|
||||
* incident required: `cap=8192MB, N watchdog kills/24h → raise --max-rss`.
|
||||
*
|
||||
* UNIONS two sources so it's authoritative for BOTH worker modes (CODEX #5):
|
||||
* - SUPERVISED workers: supervisor audit `worker_exited likely_cause=rss_watchdog`,
|
||||
* read cross-week (CODEX #7) so a Mon read doesn't lose a Sun loop.
|
||||
* - BARE `gbrain jobs work`: NO supervisor event is written; the only trace is
|
||||
* `minion_jobs.error_text = 'aborted: watchdog'` (the same source queue_health
|
||||
* subcheck 3 reads). Reading supervisor-only would miss bare workers entirely
|
||||
* and the queue_health cross-reference would point at an unemitted check.
|
||||
*
|
||||
* Cap (CODEX #6): the breaker alert stamps `max_rss_mb`, but a fail from
|
||||
* oomKills>=5 spread over 24h may have no breaker event → no stamped cap. Fall
|
||||
* back to `resolveDefaultMaxRssMb()` so the message always renders a number.
|
||||
*
|
||||
* Returns null when the worker never OOM'd (don't warn installs that never hit
|
||||
* it). Pure-ish: filesystem audit read + one minion_jobs count; no process.exit.
|
||||
* Exported so `test/doctor-worker-oom-loop.test.ts` drives it directly.
|
||||
*/
|
||||
export async function computeWorkerOomLoopCheck(
|
||||
engine: BrainEngine | null,
|
||||
): Promise<Check | null> {
|
||||
let supervisorKills = 0;
|
||||
let capFromBreaker: number | null = null;
|
||||
let breakerTripped = false;
|
||||
try {
|
||||
const { readRecentSupervisorEvents, summarizeCrashes } = await import(
|
||||
'../core/minions/handlers/supervisor-audit.ts'
|
||||
);
|
||||
const events = readRecentSupervisorEvents(24);
|
||||
supervisorKills = summarizeCrashes(events).by_cause.rss_watchdog;
|
||||
// Latest rss_watchdog_loop breaker alert carries the cap the supervisor
|
||||
// spawned with (supervisor.ts:521); its presence also means the breaker
|
||||
// tripped. Walk all events; last one wins for the cap.
|
||||
for (const e of events) {
|
||||
const row = e as Record<string, unknown>;
|
||||
if (e.event === 'health_warn' && row.reason === 'rss_watchdog_loop') {
|
||||
breakerTripped = true;
|
||||
const cap = Number(row.max_rss_mb);
|
||||
if (Number.isFinite(cap) && cap > 0) capFromBreaker = cap;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// supervisor-audit read is best-effort; fall through to minion_jobs.
|
||||
}
|
||||
|
||||
let bareWorkerKills = 0;
|
||||
if (engine && engine.kind !== 'pglite') {
|
||||
try {
|
||||
const sql = db.getConnection();
|
||||
const rows: Array<{ cnt: number }> = await sql`
|
||||
SELECT count(*)::int AS cnt
|
||||
FROM minion_jobs
|
||||
WHERE status IN ('dead', 'failed')
|
||||
AND finished_at > now() - interval '24 hours'
|
||||
AND error_text = 'aborted: watchdog'
|
||||
`;
|
||||
bareWorkerKills = rows[0]?.cnt ?? 0;
|
||||
} catch {
|
||||
// minion_jobs may not exist on a fresh brain; best-effort.
|
||||
}
|
||||
}
|
||||
|
||||
// De-dup note (CODEX #5 accepted trade-off): a supervised watchdog kill aborts
|
||||
// in-flight jobs, so it can show in BOTH counts. We accept slight over-count
|
||||
// rather than miss bare workers — the signal is "is it OOM-looping," not an
|
||||
// exact tally. `details` keeps the two sources separate for honesty.
|
||||
const oomKills = supervisorKills + bareWorkerKills;
|
||||
if (oomKills < 1 && !breakerTripped) return null;
|
||||
|
||||
let capMb: number;
|
||||
let capSource: 'breaker' | 'default';
|
||||
if (capFromBreaker !== null) {
|
||||
capMb = capFromBreaker;
|
||||
capSource = 'breaker';
|
||||
} else {
|
||||
let def = 16384;
|
||||
try {
|
||||
const { resolveDefaultMaxRssMb } = await import('../core/minions/rss-default.ts');
|
||||
def = resolveDefaultMaxRssMb();
|
||||
} catch {
|
||||
// keep the conservative ceiling fallback.
|
||||
}
|
||||
capMb = def;
|
||||
capSource = 'default';
|
||||
}
|
||||
|
||||
const fixHint =
|
||||
'raise --max-rss (gbrain jobs work --max-rss <bigger>; auto-sizes to min(0.5×RAM,16GB))';
|
||||
const capLabel = capSource === 'breaker' ? `cap=${capMb}MB` : `cap≈${capMb}MB (auto-sized default)`;
|
||||
const status: Check['status'] = breakerTripped || oomKills >= 5 ? 'fail' : 'warn';
|
||||
return {
|
||||
name: 'worker_oom_loop',
|
||||
status,
|
||||
message:
|
||||
`Worker OOM-looping: ${capLabel}, ${oomKills} watchdog kill(s)/24h → ${fixHint}. ` +
|
||||
`Peak RSS: see worker stderr.`,
|
||||
details: {
|
||||
oom_kills: oomKills,
|
||||
supervisor_kills: supervisorKills,
|
||||
bare_worker_kills: bareWorkerKills,
|
||||
cap_mb: capMb,
|
||||
cap_source: capSource,
|
||||
breaker_tripped: breakerTripped,
|
||||
fix_hint: fixHint,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* issue #1685 (GAP B) — DB pool reap health (Postgres-only).
|
||||
*
|
||||
* Answers the #1685 line "DB pool reaped N times/hr AND not auto-recovering"
|
||||
* that no existing signal expresses. Reads the pool-recovery audit
|
||||
* (`reconnect()` emits reap_detected / reconnect_succeeded / reconnect_failed):
|
||||
* - fail: reaps>0 AND reconnect failures>0 → the pool is being reaped and
|
||||
* rebuilds are throwing (genuinely not recovering).
|
||||
* - warn: reaps>=10/hr, all recovered → pooler thrash (self-heal works but the
|
||||
* cap is likely too low / concurrency too high).
|
||||
* - else: null (quiet — a few reaps that all recovered is normal).
|
||||
*
|
||||
* Returns null on PGLite / no engine / audit-read failure. Exported so
|
||||
* `test/doctor-pool-reap-health.test.ts` drives it directly.
|
||||
*/
|
||||
export async function computePoolReapHealthCheck(
|
||||
engine: BrainEngine | null,
|
||||
): Promise<Check | null> {
|
||||
if (!engine || engine.kind === 'pglite') return null;
|
||||
let r: { reaps: number; recoveries: number; failures: number };
|
||||
try {
|
||||
const { readRecentPoolRecoveries } = await import('../core/audit/pool-recovery-audit.ts');
|
||||
r = readRecentPoolRecoveries(1);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (r.reaps > 0 && r.failures > 0) {
|
||||
const fix = 'check DB reachability / credentials (reconnect is throwing)';
|
||||
return {
|
||||
name: 'pool_reap_health',
|
||||
status: 'fail',
|
||||
message:
|
||||
`DB pool reaped ${r.reaps}× and reconnect FAILED ${r.failures}× in last hour ` +
|
||||
`— not auto-recovering; ${fix}.`,
|
||||
details: { reaps: r.reaps, recoveries: r.recoveries, failures: r.failures, fix_hint: fix },
|
||||
};
|
||||
}
|
||||
if (r.reaps >= 10) {
|
||||
const fix = 'raise --max-rss or reduce worker concurrency (pooler thrash)';
|
||||
return {
|
||||
name: 'pool_reap_health',
|
||||
status: 'warn',
|
||||
message:
|
||||
`DB pool reaped ${r.reaps}× in last hour (self-heal recovered each) ` +
|
||||
`— ${fix}.`,
|
||||
details: { reaps: r.reaps, recoveries: r.recoveries, failures: r.failures, fix_hint: fix },
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function buildChecks(
|
||||
engine: BrainEngine | null,
|
||||
args: string[],
|
||||
@@ -3446,7 +3618,7 @@ export async function buildChecks(
|
||||
// shape is the right contract.
|
||||
const summary = summarizeCrashes(events);
|
||||
const crashes24h = summary.total;
|
||||
const causeStr = `runtime=${summary.by_cause.runtime_error} oom=${summary.by_cause.oom_or_external_kill} unknown=${summary.by_cause.unknown} legacy=${summary.by_cause.legacy}`;
|
||||
const causeStr = `runtime=${summary.by_cause.runtime_error} oom=${summary.by_cause.oom_or_external_kill} rss=${summary.by_cause.rss_watchdog} unknown=${summary.by_cause.unknown} legacy=${summary.by_cause.legacy}${summary.by_cause.rss_watchdog > 0 ? ' (see worker_oom_loop)' : ''}`;
|
||||
const maxCrashesEvent = events.filter(e => e.event === 'max_crashes_exceeded').pop() ?? null;
|
||||
|
||||
// Only surface a Check if the supervisor was ever observed (stops the
|
||||
@@ -3485,6 +3657,27 @@ export async function buildChecks(
|
||||
// Audit read / import failure is best-effort; skip silently.
|
||||
}
|
||||
|
||||
// 3b-quater. Worker OOM-loop (issue #1685 GAP A) — the single authoritative
|
||||
// "is the worker OOM-looping" line, unioning supervised (supervisor audit)
|
||||
// and bare-worker (minion_jobs watchdog-abort) kills. Returns null when the
|
||||
// worker never OOM'd, so clean installs see nothing.
|
||||
try {
|
||||
const oomCheck = await computeWorkerOomLoopCheck(engine);
|
||||
if (oomCheck) checks.push(oomCheck);
|
||||
} catch {
|
||||
// best-effort.
|
||||
}
|
||||
|
||||
// 3b-quinquies. DB pool reap health (issue #1685 GAP B) — Postgres pooler
|
||||
// reap frequency + recovered-vs-stuck split. Quiet unless reaps thrash or
|
||||
// reconnect is failing.
|
||||
try {
|
||||
const reapCheck = await computePoolReapHealthCheck(engine);
|
||||
if (reapCheck) checks.push(reapCheck);
|
||||
} catch {
|
||||
// best-effort.
|
||||
}
|
||||
|
||||
// 3b-tris. Stub-guard fire count (last 24h). The v0.34.5 stub guard in
|
||||
// fence-write.ts refuses to spawn unprefixed entity pages (e.g. bare
|
||||
// `alice.md` at brain root). Each fire is appended to
|
||||
@@ -5762,9 +5955,8 @@ export async function buildChecks(
|
||||
if (rssKillCount > 0) {
|
||||
problems.push(
|
||||
`${rssKillCount} job(s) dead-lettered for RSS-watchdog memory-limit kills in last 24h. ` +
|
||||
`v0.22.14 changed the bare-worker --max-rss default from 0 (off) to 2048 MB. ` +
|
||||
`Fix: raise the limit (e.g. \`gbrain jobs work --max-rss 4096\`) or opt out (\`--max-rss 0\`). ` +
|
||||
`See skills/migrations/v0.22.14.md.`
|
||||
`→ see worker_oom_loop for the cap + fix (the authoritative OOM-loop signal).`
|
||||
);
|
||||
}
|
||||
if (promptTooLongCount > 0) {
|
||||
@@ -6327,6 +6519,25 @@ function outputResults(checks: Check[], json: boolean): boolean {
|
||||
|
||||
console.log('\nGBrain Health Check');
|
||||
console.log('===================');
|
||||
|
||||
// #1685 GAP C — cause-ranked summary so the operator reads the root cause
|
||||
// first instead of scrolling the full list. Caps at 5; clean brains skip it.
|
||||
const topIssues = report.top_issues ?? [];
|
||||
if (topIssues.length > 0) {
|
||||
console.log('');
|
||||
console.log('Top issues (ranked by cause):');
|
||||
const shown = topIssues.slice(0, 5);
|
||||
for (const issue of shown) {
|
||||
const icon = issue.status === 'fail' ? 'FAIL' : 'WARN';
|
||||
const dn = issue.downstream_of ? ` (likely downstream of ${issue.downstream_of})` : '';
|
||||
console.log(` [${icon}] ${issue.name}${dn} → ${issue.fix}`);
|
||||
}
|
||||
if (topIssues.length > shown.length) {
|
||||
console.log(` +${topIssues.length - shown.length} more — see full list below`);
|
||||
}
|
||||
console.log('');
|
||||
}
|
||||
|
||||
for (const c of report.checks) {
|
||||
const icon = c.status === 'ok' ? 'OK' : c.status === 'warn' ? 'WARN' : 'FAIL';
|
||||
console.log(` [${icon}] ${c.name}: ${c.message}`);
|
||||
|
||||
@@ -138,11 +138,13 @@ export const OPS_CHECK_NAMES: ReadonlySet<string> = new Set([
|
||||
'rls',
|
||||
'rls_event_trigger',
|
||||
'search_mode',
|
||||
'pool_reap_health',
|
||||
'stale_locks',
|
||||
'subagent_capability',
|
||||
'subagent_health',
|
||||
'supervisor',
|
||||
'sync_consolidation',
|
||||
'worker_oom_loop',
|
||||
'ze_embedding_health',
|
||||
]);
|
||||
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
/**
|
||||
* issue #1685 (GAP C) — cause-ranked doctor issues.
|
||||
*
|
||||
* The #1685 posture ask: `gbrain doctor` is the single health truth, and it
|
||||
* surfaces CAUSE before symptoms. During the #1678 incident the loud lines were
|
||||
* all downstream DB-cascade noise (CONNECTION_ENDED, lock-renewal-failed) while
|
||||
* the one true cause (RSS-watchdog OOM kill) scrolled by once. This module ranks
|
||||
* the non-ok checks so the operator reads root causes first.
|
||||
*
|
||||
* HONESTY CONTRACT (CODEX #9): two checks both failing does NOT prove one caused
|
||||
* the other. So:
|
||||
* - Tier membership (root vs symptom) is ORDERING ONLY. It sorts roots above
|
||||
* symptoms; it asserts NO causality.
|
||||
* - `downstream_of` — the one place we DO claim a causal link — is set ONLY
|
||||
* from a small map of KNOWN, grounded edges, AND only when the named root is
|
||||
* itself in the failing set. It is deliberately NOT a root×symptom cartesian.
|
||||
* "Everything failing is downstream of every root" is the false-precision we
|
||||
* refuse to ship.
|
||||
*
|
||||
* Pure: no I/O, no engine, no process.exit. Unit-tested directly by
|
||||
* `test/doctor-cause-rank.test.ts`, including the drift guard that every name in
|
||||
* the cause graph still exists in `doctor-categories.ts` (DECISION 4A).
|
||||
*/
|
||||
|
||||
import {
|
||||
BRAIN_CHECK_NAMES,
|
||||
SKILL_CHECK_NAMES,
|
||||
OPS_CHECK_NAMES,
|
||||
META_CHECK_NAMES,
|
||||
} from './doctor-categories.ts';
|
||||
|
||||
/** Minimal structural shape of a doctor Check that ranking needs. */
|
||||
export interface RankableCheck {
|
||||
name: string;
|
||||
status: 'ok' | 'warn' | 'fail';
|
||||
message: string;
|
||||
details?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface RankedIssue {
|
||||
name: string;
|
||||
status: 'warn' | 'fail';
|
||||
/**
|
||||
* Coarse sort bucket. `root` = a designated root-cause check; `symptom` =
|
||||
* everything else (NOT a proof that it's a downstream effect — just "not on
|
||||
* the root-cause list"). The precise causal claim lives in `downstream_of`.
|
||||
*/
|
||||
tier: 'root' | 'symptom';
|
||||
/**
|
||||
* Set ONLY for a known causal edge whose root is also failing. Absent
|
||||
* otherwise — we never invent causality from co-occurrence.
|
||||
*/
|
||||
downstream_of?: string;
|
||||
/** One-line fix. Prefers `details.fix_hint`; falls back to the check message. */
|
||||
fix: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks that, when failing, are usually the DISEASE. Sorted to the top so the
|
||||
* operator reads the cause first. Membership is ORDERING ONLY (CODEX #9).
|
||||
*/
|
||||
export const ROOT_CAUSE_CHECKS: ReadonlySet<string> = new Set([
|
||||
'worker_oom_loop',
|
||||
'pool_reap_health',
|
||||
'connection',
|
||||
'sync_freshness',
|
||||
'schema_version',
|
||||
]);
|
||||
|
||||
/**
|
||||
* Checks that are commonly DOWNSTREAM noise during an incident. Sorted below
|
||||
* roots. Ordering only — not a causal claim.
|
||||
*/
|
||||
export const SYMPTOM_CHECKS: ReadonlySet<string> = new Set([
|
||||
'queue_health',
|
||||
'batch_retry_health',
|
||||
'supervisor',
|
||||
'stale_locks',
|
||||
]);
|
||||
|
||||
/**
|
||||
* KNOWN causal edges (symptom → root). `downstream_of` is set ONLY from this
|
||||
* map AND ONLY when the named root is itself failing. Each edge is a real,
|
||||
* grounded link, not a taxonomy guess:
|
||||
* - queue_health → worker_oom_loop: an RSS-watchdog OOM kill aborts in-flight
|
||||
* jobs; queue_health reads the SAME `error_text='aborted: watchdog'` rows
|
||||
* that worker_oom_loop counts for bare workers.
|
||||
* - supervisor → worker_oom_loop: the watchdog drain is a supervisor
|
||||
* `worker_exited likely_cause=rss_watchdog` — the exact event
|
||||
* worker_oom_loop's supervised half counts.
|
||||
*/
|
||||
const DOWNSTREAM_EDGES: Readonly<Record<string, string>> = {
|
||||
queue_health: 'worker_oom_loop',
|
||||
supervisor: 'worker_oom_loop',
|
||||
};
|
||||
|
||||
function tierOf(name: string): 'root' | 'symptom' {
|
||||
return ROOT_CAUSE_CHECKS.has(name) ? 'root' : 'symptom';
|
||||
}
|
||||
|
||||
/**
|
||||
* Rank non-ok checks: fail before warn, root before symptom, then name
|
||||
* (deterministic). Returns the full ranked list; the renderer caps to top-N.
|
||||
*/
|
||||
export function rankIssues(checks: RankableCheck[]): RankedIssue[] {
|
||||
const failing = checks.filter((c) => c.status !== 'ok');
|
||||
const failingNames = new Set(failing.map((c) => c.name));
|
||||
|
||||
const issues: RankedIssue[] = failing.map((c) => {
|
||||
const root = DOWNSTREAM_EDGES[c.name];
|
||||
const downstream_of = root && failingNames.has(root) ? root : undefined;
|
||||
const hint = c.details?.fix_hint;
|
||||
const fix =
|
||||
typeof hint === 'string' && hint.trim().length > 0 ? hint : c.message;
|
||||
return {
|
||||
name: c.name,
|
||||
status: c.status as 'warn' | 'fail',
|
||||
tier: tierOf(c.name),
|
||||
...(downstream_of ? { downstream_of } : {}),
|
||||
fix,
|
||||
};
|
||||
});
|
||||
|
||||
const statusRank = (s: string): number => (s === 'fail' ? 0 : 1);
|
||||
const tierRank = (t: string): number => (t === 'root' ? 0 : 1);
|
||||
issues.sort(
|
||||
(a, b) =>
|
||||
statusRank(a.status) - statusRank(b.status) ||
|
||||
tierRank(a.tier) - tierRank(b.tier) ||
|
||||
a.name.localeCompare(b.name),
|
||||
);
|
||||
return issues;
|
||||
}
|
||||
|
||||
/** Every name referenced by the cause graph (tiers). Drift-guard target (4A). */
|
||||
export const CAUSE_GRAPH_NAMES: ReadonlySet<string> = new Set([
|
||||
...ROOT_CAUSE_CHECKS,
|
||||
...SYMPTOM_CHECKS,
|
||||
]);
|
||||
|
||||
/** Union of all category-known check names — the drift-guard comparison set. */
|
||||
export function allKnownCheckNames(): ReadonlySet<string> {
|
||||
return new Set<string>([
|
||||
...BRAIN_CHECK_NAMES,
|
||||
...SKILL_CHECK_NAMES,
|
||||
...OPS_CHECK_NAMES,
|
||||
...META_CHECK_NAMES,
|
||||
]);
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
/**
|
||||
* #1685 GAP C — cause-ranked doctor issues. Pure unit tests.
|
||||
*
|
||||
* Covers: fail-before-warn + root-before-symptom ordering, evidence-gated
|
||||
* downstream_of (NEVER from co-occurrence alone — CODEX #9), fix-hint
|
||||
* preference, and the DECISION 4A drift guard (every cause-graph name still
|
||||
* exists in doctor-categories).
|
||||
*/
|
||||
import { describe, it, expect } from 'bun:test';
|
||||
import {
|
||||
rankIssues,
|
||||
ROOT_CAUSE_CHECKS,
|
||||
SYMPTOM_CHECKS,
|
||||
CAUSE_GRAPH_NAMES,
|
||||
allKnownCheckNames,
|
||||
type RankableCheck,
|
||||
} from '../src/core/doctor-cause-rank.ts';
|
||||
|
||||
const ok = (name: string): RankableCheck => ({ name, status: 'ok', message: 'fine' });
|
||||
const warn = (name: string, msg = 'warned'): RankableCheck => ({ name, status: 'warn', message: msg });
|
||||
const fail = (name: string, msg = 'failed'): RankableCheck => ({ name, status: 'fail', message: msg });
|
||||
|
||||
describe('rankIssues', () => {
|
||||
it('drops ok checks, returns [] when everything is healthy', () => {
|
||||
expect(rankIssues([ok('connection'), ok('queue_health')])).toEqual([]);
|
||||
});
|
||||
|
||||
it('orders fail before warn', () => {
|
||||
const out = rankIssues([warn('stale_locks'), fail('schema_version')]);
|
||||
expect(out.map((i) => i.name)).toEqual(['schema_version', 'stale_locks']);
|
||||
});
|
||||
|
||||
it('orders root before symptom within the same status', () => {
|
||||
// queue_health (symptom) + worker_oom_loop (root), both fail.
|
||||
const out = rankIssues([fail('queue_health'), fail('worker_oom_loop')]);
|
||||
expect(out.map((i) => i.name)).toEqual(['worker_oom_loop', 'queue_health']);
|
||||
expect(out[0].tier).toBe('root');
|
||||
expect(out[1].tier).toBe('symptom');
|
||||
});
|
||||
|
||||
it('tags downstream_of ONLY when the named root is itself failing', () => {
|
||||
const both = rankIssues([fail('worker_oom_loop'), fail('queue_health')]);
|
||||
const q1 = both.find((i) => i.name === 'queue_health')!;
|
||||
expect(q1.downstream_of).toBe('worker_oom_loop');
|
||||
});
|
||||
|
||||
it('does NOT tag downstream_of when the root is absent (no co-occurrence guess)', () => {
|
||||
const out = rankIssues([fail('queue_health')]); // worker_oom_loop not failing
|
||||
const q = out.find((i) => i.name === 'queue_health')!;
|
||||
expect(q.downstream_of).toBeUndefined();
|
||||
});
|
||||
|
||||
it('does NOT tag downstream_of from a generic root×symptom cartesian', () => {
|
||||
// schema_version is a root and stale_locks is a symptom, but there is NO
|
||||
// declared causal edge between them — co-occurrence must not invent one.
|
||||
const out = rankIssues([fail('schema_version'), fail('stale_locks')]);
|
||||
const s = out.find((i) => i.name === 'stale_locks')!;
|
||||
expect(s.downstream_of).toBeUndefined();
|
||||
});
|
||||
|
||||
it('uses details.fix_hint when present, else the message', () => {
|
||||
const out = rankIssues([
|
||||
{ name: 'worker_oom_loop', status: 'fail', message: 'long message', details: { fix_hint: 'raise --max-rss' } },
|
||||
warn('orphan_ratio', 'too many orphans'),
|
||||
]);
|
||||
expect(out.find((i) => i.name === 'worker_oom_loop')!.fix).toBe('raise --max-rss');
|
||||
expect(out.find((i) => i.name === 'orphan_ratio')!.fix).toBe('too many orphans');
|
||||
});
|
||||
|
||||
it('is deterministic (name tiebreak) for same status+tier', () => {
|
||||
const out = rankIssues([warn('zeta_unknown'), warn('alpha_unknown')]);
|
||||
expect(out.map((i) => i.name)).toEqual(['alpha_unknown', 'zeta_unknown']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('DECISION 4A drift guard', () => {
|
||||
it('every cause-graph name exists in doctor-categories known names', () => {
|
||||
const known = allKnownCheckNames();
|
||||
const missing = [...CAUSE_GRAPH_NAMES].filter((n) => !known.has(n));
|
||||
expect(missing).toEqual([]);
|
||||
});
|
||||
|
||||
it('root and symptom sets are disjoint', () => {
|
||||
const overlap = [...ROOT_CAUSE_CHECKS].filter((n) => SYMPTOM_CHECKS.has(n));
|
||||
expect(overlap).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,64 @@
|
||||
// #1685 GAP B — pool_reap_health doctor check.
|
||||
//
|
||||
// computePoolReapHealthCheck only touches engine.kind + the pool-recovery audit
|
||||
// (filesystem), so a minimal `{ kind: 'postgres' }` stub drives it hermetically.
|
||||
|
||||
import { describe, expect, test, beforeEach, afterEach } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { withEnv } from './helpers/with-env.ts';
|
||||
import { logPoolRecovery } from '../src/core/audit/pool-recovery-audit.ts';
|
||||
import { computePoolReapHealthCheck } from '../src/commands/doctor.ts';
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const pg = { kind: 'postgres' } as any;
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const pglite = { kind: 'pglite' } as any;
|
||||
|
||||
let tmpDir: string;
|
||||
beforeEach(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pool-reap-health-'));
|
||||
});
|
||||
afterEach(() => {
|
||||
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* best-effort */ }
|
||||
});
|
||||
|
||||
describe('computePoolReapHealthCheck', () => {
|
||||
test('null on PGLite (no pool) and on null engine', async () => {
|
||||
expect(await computePoolReapHealthCheck(pglite)).toBeNull();
|
||||
expect(await computePoolReapHealthCheck(null)).toBeNull();
|
||||
});
|
||||
|
||||
test('fail when reaps>0 AND reconnect failed (not auto-recovering)', async () => {
|
||||
await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
|
||||
logPoolRecovery('reap_detected');
|
||||
logPoolRecovery('reconnect_failed', new Error('EHOSTUNREACH'));
|
||||
const c = await computePoolReapHealthCheck(pg);
|
||||
expect(c?.status).toBe('fail');
|
||||
expect(c?.message).toContain('not auto-recovering');
|
||||
expect(c?.name).toBe('pool_reap_health');
|
||||
});
|
||||
});
|
||||
|
||||
test('warn on pooler thrash (>=10 reaps all recovered)', async () => {
|
||||
await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
|
||||
for (let i = 0; i < 12; i++) {
|
||||
logPoolRecovery('reap_detected');
|
||||
logPoolRecovery('reconnect_succeeded');
|
||||
}
|
||||
const c = await computePoolReapHealthCheck(pg);
|
||||
expect(c?.status).toBe('warn');
|
||||
expect(c?.message).toContain('12×');
|
||||
});
|
||||
});
|
||||
|
||||
test('null (quiet) when a few reaps all recovered', async () => {
|
||||
await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
|
||||
logPoolRecovery('reap_detected');
|
||||
logPoolRecovery('reconnect_succeeded');
|
||||
const c = await computePoolReapHealthCheck(pg);
|
||||
expect(c).toBeNull();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,101 @@
|
||||
// #1685 GAP A — worker_oom_loop doctor check.
|
||||
//
|
||||
// Hermetic: writes synthetic supervisor audit JSONL into a GBRAIN_AUDIT_DIR
|
||||
// tempdir and drives computeWorkerOomLoopCheck with engine=null (supervised
|
||||
// half + cap logic + thresholds; the minion_jobs bare-worker branch is
|
||||
// Postgres-only and mirrors the queue_health subcheck-3 query covered by E2E).
|
||||
// Also pins the cross-week supervisor reader (CODEX #7).
|
||||
|
||||
import { describe, expect, test, beforeEach, afterEach } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { withEnv } from './helpers/with-env.ts';
|
||||
import { computeWorkerOomLoopCheck } from '../src/commands/doctor.ts';
|
||||
import {
|
||||
computeSupervisorAuditFilename,
|
||||
readRecentSupervisorEvents,
|
||||
} from '../src/core/minions/handlers/supervisor-audit.ts';
|
||||
|
||||
let tmpDir: string;
|
||||
beforeEach(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'worker-oom-loop-'));
|
||||
});
|
||||
afterEach(() => {
|
||||
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* best-effort */ }
|
||||
});
|
||||
|
||||
function writeSupervisorRows(rows: object[], fileDate = new Date()): void {
|
||||
const file = path.join(tmpDir, computeSupervisorAuditFilename(fileDate));
|
||||
fs.writeFileSync(file, rows.map((r) => JSON.stringify(r)).join('\n') + '\n', 'utf8');
|
||||
}
|
||||
|
||||
const nowIso = () => new Date().toISOString();
|
||||
const rssKill = () => ({ event: 'worker_exited', ts: nowIso(), likely_cause: 'rss_watchdog', code: 12 });
|
||||
const breaker = (cap: number) => ({ event: 'health_warn', ts: nowIso(), reason: 'rss_watchdog_loop', max_rss_mb: cap });
|
||||
|
||||
describe('computeWorkerOomLoopCheck', () => {
|
||||
test('fail with breaker-stamped cap when the breaker tripped', async () => {
|
||||
await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
|
||||
writeSupervisorRows([rssKill(), rssKill(), rssKill(), rssKill(), rssKill(), rssKill(), breaker(2048)]);
|
||||
const c = await computeWorkerOomLoopCheck(null);
|
||||
expect(c?.status).toBe('fail');
|
||||
expect(c?.name).toBe('worker_oom_loop');
|
||||
expect(c?.message).toContain('cap=2048MB');
|
||||
expect(c?.message).toContain('raise --max-rss');
|
||||
expect(c?.details?.oom_kills).toBe(6);
|
||||
expect(c?.details?.supervisor_kills).toBe(6);
|
||||
expect(c?.details?.bare_worker_kills).toBe(0);
|
||||
expect(c?.details?.cap_source).toBe('breaker');
|
||||
expect(c?.details?.breaker_tripped).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
test('warn with auto-sized cap fallback when no breaker event stamped a cap (CODEX #6)', async () => {
|
||||
await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
|
||||
writeSupervisorRows([rssKill(), rssKill()]);
|
||||
const c = await computeWorkerOomLoopCheck(null);
|
||||
expect(c?.status).toBe('warn');
|
||||
expect(c?.details?.cap_source).toBe('default');
|
||||
expect(c?.message).toContain('auto-sized default');
|
||||
expect(c?.details?.oom_kills).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
test('fail at >=5 kills even without a breaker event', async () => {
|
||||
await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
|
||||
writeSupervisorRows([rssKill(), rssKill(), rssKill(), rssKill(), rssKill()]);
|
||||
const c = await computeWorkerOomLoopCheck(null);
|
||||
expect(c?.status).toBe('fail');
|
||||
expect(c?.details?.breaker_tripped).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
test('null when the worker never OOM-looped', async () => {
|
||||
await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
|
||||
writeSupervisorRows([{ event: 'started', ts: nowIso() }]);
|
||||
expect(await computeWorkerOomLoopCheck(null)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
test('null on an empty audit dir', async () => {
|
||||
await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
|
||||
expect(await computeWorkerOomLoopCheck(null)).toBeNull();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('readRecentSupervisorEvents — cross-week (CODEX #7)', () => {
|
||||
test('reads a within-24h event from the PREVIOUS ISO-week file', async () => {
|
||||
await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
|
||||
// Event timestamped within the 24h window but written into last week's
|
||||
// file (the Monday-reads-Sunday case). The single-file reader would miss
|
||||
// it; the cross-week reader must find it.
|
||||
const prevWeekFileDate = new Date(Date.now() - 7 * 86400000);
|
||||
writeSupervisorRows([rssKill()], prevWeekFileDate);
|
||||
const events = readRecentSupervisorEvents(24);
|
||||
expect(events.length).toBe(1);
|
||||
expect(events[0].event).toBe('worker_exited');
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user