mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
* feat(minions): pool-recovery audit + reconnect reason-threading + shared drain helper (#1685 GAP B, 5A) - pool-recovery-audit.ts: reap_detected (CONNECTION_ENDED) vs reconnect_other; recovered/failed split - postgres-engine reconnect(ctx?) classifies the triggering error so only true pooler reaps are tagged (CODEX #8) - retry.ts reconnect callback widened to thread the error; retry-matcher isConnectionEndedError - runExtractAtomsDrainForSource shared helper (cycleLockIdFor + withRefreshingLock) — one drain path (5A) - supervisor-audit readRecentSupervisorEvents (current+prev ISO week, CODEX #7) - extract-atoms-drain PROTECTED; autopilot.auto_drain.* config keys * feat(doctor): worker_oom_loop + pool_reap_health checks + cause-ranked top_issues (#1685 GAP A/B/C) - computeWorkerOomLoopCheck: unions supervisor rss_watchdog + minion_jobs watchdog-abort (CODEX #5), cap fallback to resolveDefaultMaxRssMb (CODEX #6) - computePoolReapHealthCheck: reaps-not-recovering fail, thrash warn - doctor-cause-rank rankIssues: tier ordering + grounded downstream_of (CODEX #9) + drift guard (4A) - supervisor causeStr + queue_health cross-reference worker_oom_loop (DRY 1C) - register both checks in doctor-categories ops * feat(autopilot): per-source extract_atoms auto-drain + handler + dream --drain refactor (#1685 GAP D) - autopilot per-source gate: enabled + !packDeclares + backlog>threshold + daily cap; time-sloted idempotency key (CODEX #2) - extract-atoms-drain Minion handler (thin wrapper, LockUnavailableError -> deferred) - dream --drain routes through the shared helper (5A) * chore: bump version and changelog (v0.42.12.0) #1685 brain-health-as-solved-problem: cause-ranked doctor, worker_oom_loop line, per-source auto-drain, pool-reap health. Layers on #1678/#1735. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * docs(todos): file #1685 GAP E + remote-path follow-ups (v0.42.12.0) * fix(#1685): pre-landing review — multi-source auto-drain, honest pool-reap signal, lock-renewal reap labeling - autopilot: drop maxWaiting (coalesces by name+queue not source → only one source drained + cap over-count); pre-check idempotency key so only genuinely-new sources submit+count - pool_reap_health: fail on reconnect FAILURES (the real signal), not reaps>0&&failures>0 (false causality when a recovered reap + unrelated failure co-occur) - lock-renewal-tick threads its triggering error to reconnect() so a CONNECTION_ENDED pooler reap is labeled reap_detected not reconnect_other (pool_reap_health now fires for the #1678 incident path) * chore: re-version v0.42.12.0 → v0.42.16.0 (#1685) Slot collision avoidance per queue. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * docs: restore slim CLAUDE.md + move #1685 entries to KEY_FILES.md (fix check:doc-history) The master merge wrongly kept the pre-restructure 577KB CLAUDE.md; the check:doc-history guard caps it at 60KB. Take master's slim CLAUDE.md and record the #1685 files (doctor-cause-rank, pool-recovery-audit, worker_oom_loop + pool_reap_health checks, auto-drain, 5A helper) as current-state prose in docs/architecture/KEY_FILES.md (no release markers). llms regenerated. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
88 lines
3.7 KiB
TypeScript
88 lines
3.7 KiB
TypeScript
/**
|
||
* #1685 GAP C — cause-ranked doctor issues. Pure unit tests.
|
||
*
|
||
* Covers: fail-before-warn + root-before-symptom ordering, evidence-gated
|
||
* downstream_of (NEVER from co-occurrence alone — CODEX #9), fix-hint
|
||
* preference, and the DECISION 4A drift guard (every cause-graph name still
|
||
* exists in doctor-categories).
|
||
*/
|
||
import { describe, it, expect } from 'bun:test';
|
||
import {
|
||
rankIssues,
|
||
ROOT_CAUSE_CHECKS,
|
||
SYMPTOM_CHECKS,
|
||
CAUSE_GRAPH_NAMES,
|
||
allKnownCheckNames,
|
||
type RankableCheck,
|
||
} from '../src/core/doctor-cause-rank.ts';
|
||
|
||
const ok = (name: string): RankableCheck => ({ name, status: 'ok', message: 'fine' });
|
||
const warn = (name: string, msg = 'warned'): RankableCheck => ({ name, status: 'warn', message: msg });
|
||
const fail = (name: string, msg = 'failed'): RankableCheck => ({ name, status: 'fail', message: msg });
|
||
|
||
describe('rankIssues', () => {
|
||
it('drops ok checks, returns [] when everything is healthy', () => {
|
||
expect(rankIssues([ok('connection'), ok('queue_health')])).toEqual([]);
|
||
});
|
||
|
||
it('orders fail before warn', () => {
|
||
const out = rankIssues([warn('stale_locks'), fail('schema_version')]);
|
||
expect(out.map((i) => i.name)).toEqual(['schema_version', 'stale_locks']);
|
||
});
|
||
|
||
it('orders root before symptom within the same status', () => {
|
||
// queue_health (symptom) + worker_oom_loop (root), both fail.
|
||
const out = rankIssues([fail('queue_health'), fail('worker_oom_loop')]);
|
||
expect(out.map((i) => i.name)).toEqual(['worker_oom_loop', 'queue_health']);
|
||
expect(out[0].tier).toBe('root');
|
||
expect(out[1].tier).toBe('symptom');
|
||
});
|
||
|
||
it('tags downstream_of ONLY when the named root is itself failing', () => {
|
||
const both = rankIssues([fail('worker_oom_loop'), fail('queue_health')]);
|
||
const q1 = both.find((i) => i.name === 'queue_health')!;
|
||
expect(q1.downstream_of).toBe('worker_oom_loop');
|
||
});
|
||
|
||
it('does NOT tag downstream_of when the root is absent (no co-occurrence guess)', () => {
|
||
const out = rankIssues([fail('queue_health')]); // worker_oom_loop not failing
|
||
const q = out.find((i) => i.name === 'queue_health')!;
|
||
expect(q.downstream_of).toBeUndefined();
|
||
});
|
||
|
||
it('does NOT tag downstream_of from a generic root×symptom cartesian', () => {
|
||
// schema_version is a root and stale_locks is a symptom, but there is NO
|
||
// declared causal edge between them — co-occurrence must not invent one.
|
||
const out = rankIssues([fail('schema_version'), fail('stale_locks')]);
|
||
const s = out.find((i) => i.name === 'stale_locks')!;
|
||
expect(s.downstream_of).toBeUndefined();
|
||
});
|
||
|
||
it('uses details.fix_hint when present, else the message', () => {
|
||
const out = rankIssues([
|
||
{ name: 'worker_oom_loop', status: 'fail', message: 'long message', details: { fix_hint: 'raise --max-rss' } },
|
||
warn('orphan_ratio', 'too many orphans'),
|
||
]);
|
||
expect(out.find((i) => i.name === 'worker_oom_loop')!.fix).toBe('raise --max-rss');
|
||
expect(out.find((i) => i.name === 'orphan_ratio')!.fix).toBe('too many orphans');
|
||
});
|
||
|
||
it('is deterministic (name tiebreak) for same status+tier', () => {
|
||
const out = rankIssues([warn('zeta_unknown'), warn('alpha_unknown')]);
|
||
expect(out.map((i) => i.name)).toEqual(['alpha_unknown', 'zeta_unknown']);
|
||
});
|
||
});
|
||
|
||
describe('DECISION 4A drift guard', () => {
|
||
it('every cause-graph name exists in doctor-categories known names', () => {
|
||
const known = allKnownCheckNames();
|
||
const missing = [...CAUSE_GRAPH_NAMES].filter((n) => !known.has(n));
|
||
expect(missing).toEqual([]);
|
||
});
|
||
|
||
it('root and symptom sets are disjoint', () => {
|
||
const overlap = [...ROOT_CAUSE_CHECKS].filter((n) => SYMPTOM_CHECKS.has(n));
|
||
expect(overlap).toEqual([]);
|
||
});
|
||
});
|