mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
* feat(minions): pool-recovery audit + reconnect reason-threading + shared drain helper (#1685 GAP B, 5A) - pool-recovery-audit.ts: reap_detected (CONNECTION_ENDED) vs reconnect_other; recovered/failed split - postgres-engine reconnect(ctx?) classifies the triggering error so only true pooler reaps are tagged (CODEX #8) - retry.ts reconnect callback widened to thread the error; retry-matcher isConnectionEndedError - runExtractAtomsDrainForSource shared helper (cycleLockIdFor + withRefreshingLock) — one drain path (5A) - supervisor-audit readRecentSupervisorEvents (current+prev ISO week, CODEX #7) - extract-atoms-drain PROTECTED; autopilot.auto_drain.* config keys * feat(doctor): worker_oom_loop + pool_reap_health checks + cause-ranked top_issues (#1685 GAP A/B/C) - computeWorkerOomLoopCheck: unions supervisor rss_watchdog + minion_jobs watchdog-abort (CODEX #5), cap fallback to resolveDefaultMaxRssMb (CODEX #6) - computePoolReapHealthCheck: reaps-not-recovering fail, thrash warn - doctor-cause-rank rankIssues: tier ordering + grounded downstream_of (CODEX #9) + drift guard (4A) - supervisor causeStr + queue_health cross-reference worker_oom_loop (DRY 1C) - register both checks in doctor-categories ops * feat(autopilot): per-source extract_atoms auto-drain + handler + dream --drain refactor (#1685 GAP D) - autopilot per-source gate: enabled + !packDeclares + backlog>threshold + daily cap; time-sloted idempotency key (CODEX #2) - extract-atoms-drain Minion handler (thin wrapper, LockUnavailableError -> deferred) - dream --drain routes through the shared helper (5A) * chore: bump version and changelog (v0.42.12.0) #1685 brain-health-as-solved-problem: cause-ranked doctor, worker_oom_loop line, per-source auto-drain, pool-reap health. Layers on #1678/#1735. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * docs(todos): file #1685 GAP E + remote-path follow-ups (v0.42.12.0) * fix(#1685): pre-landing review — multi-source auto-drain, honest pool-reap signal, lock-renewal reap labeling - autopilot: drop maxWaiting (coalesces by name+queue not source → only one source drained + cap over-count); pre-check idempotency key so only genuinely-new sources submit+count - pool_reap_health: fail on reconnect FAILURES (the real signal), not reaps>0&&failures>0 (false causality when a recovered reap + unrelated failure co-occur) - lock-renewal-tick threads its triggering error to reconnect() so a CONNECTION_ENDED pooler reap is labeled reap_detected not reconnect_other (pool_reap_health now fires for the #1678 incident path) * chore: re-version v0.42.12.0 → v0.42.16.0 (#1685) Slot collision avoidance per queue. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * docs: restore slim CLAUDE.md + move #1685 entries to KEY_FILES.md (fix check:doc-history) The master merge wrongly kept the pre-restructure 577KB CLAUDE.md; the check:doc-history guard caps it at 60KB. Take master's slim CLAUDE.md and record the #1685 files (doctor-cause-rank, pool-recovery-audit, worker_oom_loop + pool_reap_health checks, auto-drain, 5A helper) as current-state prose in docs/architecture/KEY_FILES.md (no release markers). llms regenerated. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
87 lines
3.1 KiB
TypeScript
87 lines
3.1 KiB
TypeScript
// #1685 GAP B — pool-recovery audit JSONL primitive.
|
|
//
|
|
// Hermetic: GBRAIN_AUDIT_DIR override via withEnv; fresh tempdir per test.
|
|
|
|
import { describe, expect, test, beforeEach, afterEach } from 'bun:test';
|
|
import * as fs from 'fs';
|
|
import * as os from 'os';
|
|
import * as path from 'path';
|
|
import { withEnv } from '../helpers/with-env.ts';
|
|
import {
|
|
logPoolRecovery,
|
|
readRecentPoolRecoveries,
|
|
_poolRecoveryAuditFeatureName,
|
|
} from '../../src/core/audit/pool-recovery-audit.ts';
|
|
|
|
let tmpDir: string;
|
|
beforeEach(() => {
|
|
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pool-recovery-audit-'));
|
|
});
|
|
afterEach(() => {
|
|
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* best-effort */ }
|
|
});
|
|
|
|
describe('logPoolRecovery + readRecentPoolRecoveries', () => {
|
|
test('round-trips a reap → recovered pair into the right counters', async () => {
|
|
await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
|
|
logPoolRecovery('reap_detected', Object.assign(new Error('write CONNECTION_ENDED'), { code: 'CONNECTION_ENDED' }));
|
|
logPoolRecovery('reconnect_succeeded');
|
|
const r = readRecentPoolRecoveries(1);
|
|
expect(r.reaps).toBe(1);
|
|
expect(r.recoveries).toBe(1);
|
|
expect(r.failures).toBe(0);
|
|
expect(r.others).toBe(0);
|
|
expect(r.events).toHaveLength(2);
|
|
});
|
|
});
|
|
|
|
test('counts a reap that failed to recover (the not-auto-recovering signal)', async () => {
|
|
await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
|
|
logPoolRecovery('reap_detected');
|
|
logPoolRecovery('reconnect_failed', new Error('EHOSTUNREACH'));
|
|
const r = readRecentPoolRecoveries(1);
|
|
expect(r.reaps).toBe(1);
|
|
expect(r.failures).toBe(1);
|
|
expect(r.recoveries).toBe(0);
|
|
});
|
|
});
|
|
|
|
test('reconnect_other is tracked separately from reaps', async () => {
|
|
await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
|
|
logPoolRecovery('reconnect_other', new Error('network blip'));
|
|
logPoolRecovery('reconnect_succeeded');
|
|
const r = readRecentPoolRecoveries(1);
|
|
expect(r.reaps).toBe(0);
|
|
expect(r.others).toBe(1);
|
|
expect(r.recoveries).toBe(1);
|
|
});
|
|
});
|
|
|
|
test('redacts connection info from the error summary', async () => {
|
|
await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
|
|
logPoolRecovery(
|
|
'reconnect_failed',
|
|
new Error('could not connect to postgres://user:secret@db.example.com:5432/app (192.168.1.42)'),
|
|
);
|
|
const r = readRecentPoolRecoveries(1);
|
|
const summary = r.events[0].error_summary ?? '';
|
|
expect(summary).not.toContain('secret');
|
|
expect(summary).not.toContain('192.168.1.42');
|
|
expect(summary).toContain('<REDACTED');
|
|
});
|
|
});
|
|
|
|
test('empty dir → all-zero counters, no throw', async () => {
|
|
await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
|
|
const r = readRecentPoolRecoveries(1);
|
|
expect(r.reaps).toBe(0);
|
|
expect(r.events).toEqual([]);
|
|
expect(r.most_recent_ts).toBeNull();
|
|
});
|
|
});
|
|
|
|
test('stable feature name', () => {
|
|
expect(_poolRecoveryAuditFeatureName()).toBe('pool-recovery');
|
|
});
|
|
});
|