Files
gbrain/test/doctor-cause-rank.test.ts
T
3fe449361c v0.42.16.0 feat(doctor): brain health as a solved problem — cause-ranked doctor + OOM-loop line + auto-drain + pool-reap (#1685) (#1802)
* feat(minions): pool-recovery audit + reconnect reason-threading + shared drain helper (#1685 GAP B, 5A)

- pool-recovery-audit.ts: reap_detected (CONNECTION_ENDED) vs reconnect_other; recovered/failed split
- postgres-engine reconnect(ctx?) classifies the triggering error so only true pooler reaps are tagged (CODEX #8)
- retry.ts reconnect callback widened to thread the error; retry-matcher isConnectionEndedError
- runExtractAtomsDrainForSource shared helper (cycleLockIdFor + withRefreshingLock) — one drain path (5A)
- supervisor-audit readRecentSupervisorEvents (current+prev ISO week, CODEX #7)
- extract-atoms-drain PROTECTED; autopilot.auto_drain.* config keys

* feat(doctor): worker_oom_loop + pool_reap_health checks + cause-ranked top_issues (#1685 GAP A/B/C)

- computeWorkerOomLoopCheck: unions supervisor rss_watchdog + minion_jobs watchdog-abort (CODEX #5), cap fallback to resolveDefaultMaxRssMb (CODEX #6)
- computePoolReapHealthCheck: reaps-not-recovering fail, thrash warn
- doctor-cause-rank rankIssues: tier ordering + grounded downstream_of (CODEX #9) + drift guard (4A)
- supervisor causeStr + queue_health cross-reference worker_oom_loop (DRY 1C)
- register both checks in doctor-categories ops

* feat(autopilot): per-source extract_atoms auto-drain + handler + dream --drain refactor (#1685 GAP D)

- autopilot per-source gate: enabled + !packDeclares + backlog>threshold + daily cap; time-sloted idempotency key (CODEX #2)
- extract-atoms-drain Minion handler (thin wrapper, LockUnavailableError -> deferred)
- dream --drain routes through the shared helper (5A)

* chore: bump version and changelog (v0.42.12.0)

#1685 brain-health-as-solved-problem: cause-ranked doctor, worker_oom_loop
line, per-source auto-drain, pool-reap health. Layers on #1678/#1735.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* docs(todos): file #1685 GAP E + remote-path follow-ups (v0.42.12.0)

* fix(#1685): pre-landing review — multi-source auto-drain, honest pool-reap signal, lock-renewal reap labeling

- autopilot: drop maxWaiting (coalesces by name+queue not source → only one source drained + cap over-count); pre-check idempotency key so only genuinely-new sources submit+count
- pool_reap_health: fail on reconnect FAILURES (the real signal), not reaps>0&&failures>0 (false causality when a recovered reap + unrelated failure co-occur)
- lock-renewal-tick threads its triggering error to reconnect() so a CONNECTION_ENDED pooler reap is labeled reap_detected not reconnect_other (pool_reap_health now fires for the #1678 incident path)

* chore: re-version v0.42.12.0 → v0.42.16.0 (#1685)

Slot collision avoidance per queue.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* docs: restore slim CLAUDE.md + move #1685 entries to KEY_FILES.md (fix check:doc-history)

The master merge wrongly kept the pre-restructure 577KB CLAUDE.md; the
check:doc-history guard caps it at 60KB. Take master's slim CLAUDE.md and
record the #1685 files (doctor-cause-rank, pool-recovery-audit, worker_oom_loop
+ pool_reap_health checks, auto-drain, 5A helper) as current-state prose in
docs/architecture/KEY_FILES.md (no release markers). llms regenerated.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-03 07:27:34 -07:00

88 lines
3.7 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* #1685 GAP C — cause-ranked doctor issues. Pure unit tests.
*
* Covers: fail-before-warn + root-before-symptom ordering, evidence-gated
* downstream_of (NEVER from co-occurrence alone — CODEX #9), fix-hint
* preference, and the DECISION 4A drift guard (every cause-graph name still
* exists in doctor-categories).
*/
import { describe, it, expect } from 'bun:test';
import {
rankIssues,
ROOT_CAUSE_CHECKS,
SYMPTOM_CHECKS,
CAUSE_GRAPH_NAMES,
allKnownCheckNames,
type RankableCheck,
} from '../src/core/doctor-cause-rank.ts';
const ok = (name: string): RankableCheck => ({ name, status: 'ok', message: 'fine' });
const warn = (name: string, msg = 'warned'): RankableCheck => ({ name, status: 'warn', message: msg });
const fail = (name: string, msg = 'failed'): RankableCheck => ({ name, status: 'fail', message: msg });
describe('rankIssues', () => {
it('drops ok checks, returns [] when everything is healthy', () => {
expect(rankIssues([ok('connection'), ok('queue_health')])).toEqual([]);
});
it('orders fail before warn', () => {
const out = rankIssues([warn('stale_locks'), fail('schema_version')]);
expect(out.map((i) => i.name)).toEqual(['schema_version', 'stale_locks']);
});
it('orders root before symptom within the same status', () => {
// queue_health (symptom) + worker_oom_loop (root), both fail.
const out = rankIssues([fail('queue_health'), fail('worker_oom_loop')]);
expect(out.map((i) => i.name)).toEqual(['worker_oom_loop', 'queue_health']);
expect(out[0].tier).toBe('root');
expect(out[1].tier).toBe('symptom');
});
it('tags downstream_of ONLY when the named root is itself failing', () => {
const both = rankIssues([fail('worker_oom_loop'), fail('queue_health')]);
const q1 = both.find((i) => i.name === 'queue_health')!;
expect(q1.downstream_of).toBe('worker_oom_loop');
});
it('does NOT tag downstream_of when the root is absent (no co-occurrence guess)', () => {
const out = rankIssues([fail('queue_health')]); // worker_oom_loop not failing
const q = out.find((i) => i.name === 'queue_health')!;
expect(q.downstream_of).toBeUndefined();
});
it('does NOT tag downstream_of from a generic root×symptom cartesian', () => {
// schema_version is a root and stale_locks is a symptom, but there is NO
// declared causal edge between them — co-occurrence must not invent one.
const out = rankIssues([fail('schema_version'), fail('stale_locks')]);
const s = out.find((i) => i.name === 'stale_locks')!;
expect(s.downstream_of).toBeUndefined();
});
it('uses details.fix_hint when present, else the message', () => {
const out = rankIssues([
{ name: 'worker_oom_loop', status: 'fail', message: 'long message', details: { fix_hint: 'raise --max-rss' } },
warn('orphan_ratio', 'too many orphans'),
]);
expect(out.find((i) => i.name === 'worker_oom_loop')!.fix).toBe('raise --max-rss');
expect(out.find((i) => i.name === 'orphan_ratio')!.fix).toBe('too many orphans');
});
it('is deterministic (name tiebreak) for same status+tier', () => {
const out = rankIssues([warn('zeta_unknown'), warn('alpha_unknown')]);
expect(out.map((i) => i.name)).toEqual(['alpha_unknown', 'zeta_unknown']);
});
});
describe('DECISION 4A drift guard', () => {
it('every cause-graph name exists in doctor-categories known names', () => {
const known = allKnownCheckNames();
const missing = [...CAUSE_GRAPH_NAMES].filter((n) => !known.has(n));
expect(missing).toEqual([]);
});
it('root and symptom sets are disjoint', () => {
const overlap = [...ROOT_CAUSE_CHECKS].filter((n) => SYMPTOM_CHECKS.has(n));
expect(overlap).toEqual([]);
});
});