Files
gbrain/test/extract-atoms-drain.test.ts
T
3fe449361c v0.42.16.0 feat(doctor): brain health as a solved problem — cause-ranked doctor + OOM-loop line + auto-drain + pool-reap (#1685) (#1802)
* feat(minions): pool-recovery audit + reconnect reason-threading + shared drain helper (#1685 GAP B, 5A)

- pool-recovery-audit.ts: reap_detected (CONNECTION_ENDED) vs reconnect_other; recovered/failed split
- postgres-engine reconnect(ctx?) classifies the triggering error so only true pooler reaps are tagged (CODEX #8)
- retry.ts reconnect callback widened to thread the error; retry-matcher isConnectionEndedError
- runExtractAtomsDrainForSource shared helper (cycleLockIdFor + withRefreshingLock) — one drain path (5A)
- supervisor-audit readRecentSupervisorEvents (current+prev ISO week, CODEX #7)
- extract-atoms-drain PROTECTED; autopilot.auto_drain.* config keys

* feat(doctor): worker_oom_loop + pool_reap_health checks + cause-ranked top_issues (#1685 GAP A/B/C)

- computeWorkerOomLoopCheck: unions supervisor rss_watchdog + minion_jobs watchdog-abort (CODEX #5), cap fallback to resolveDefaultMaxRssMb (CODEX #6)
- computePoolReapHealthCheck: reaps-not-recovering fail, thrash warn
- doctor-cause-rank rankIssues: tier ordering + grounded downstream_of (CODEX #9) + drift guard (4A)
- supervisor causeStr + queue_health cross-reference worker_oom_loop (DRY 1C)
- register both checks in doctor-categories ops

* feat(autopilot): per-source extract_atoms auto-drain + handler + dream --drain refactor (#1685 GAP D)

- autopilot per-source gate: enabled + !packDeclares + backlog>threshold + daily cap; time-sloted idempotency key (CODEX #2)
- extract-atoms-drain Minion handler (thin wrapper, LockUnavailableError -> deferred)
- dream --drain routes through the shared helper (5A)

* chore: bump version and changelog (v0.42.12.0)

#1685 brain-health-as-solved-problem: cause-ranked doctor, worker_oom_loop
line, per-source auto-drain, pool-reap health. Layers on #1678/#1735.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* docs(todos): file #1685 GAP E + remote-path follow-ups (v0.42.12.0)

* fix(#1685): pre-landing review — multi-source auto-drain, honest pool-reap signal, lock-renewal reap labeling

- autopilot: drop maxWaiting (coalesces by name+queue not source → only one source drained + cap over-count); pre-check idempotency key so only genuinely-new sources submit+count
- pool_reap_health: fail on reconnect FAILURES (the real signal), not reaps>0&&failures>0 (false causality when a recovered reap + unrelated failure co-occur)
- lock-renewal-tick threads its triggering error to reconnect() so a CONNECTION_ENDED pooler reap is labeled reap_detected not reconnect_other (pool_reap_health now fires for the #1678 incident path)

* chore: re-version v0.42.12.0 → v0.42.16.0 (#1685)

Slot collision avoidance per queue.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* docs: restore slim CLAUDE.md + move #1685 entries to KEY_FILES.md (fix check:doc-history)

The master merge wrongly kept the pre-restructure 577KB CLAUDE.md; the
check:doc-history guard caps it at 60KB. Take master's slim CLAUDE.md and
record the #1685 files (doctor-cause-rank, pool-recovery-audit, worker_oom_loop
+ pool_reap_health checks, auto-drain, 5A helper) as current-state prose in
docs/architecture/KEY_FILES.md (no release markers). llms regenerated.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-03 07:27:34 -07:00

137 lines
4.9 KiB
TypeScript

/**
* issue #1678 — bounded single-hold extract_atoms drain loop.
*
* Pure-over-injected-deps, so no DB / LLM / lock primitive. Pins:
* - drains to empty (rediscovers each batch via countRemaining), stops 'drained'
* - the wallclock window bounds the loop, stops 'window' with remaining > 0
* - a zero-progress batch stops the loop (no hot loop burning budget)
* - a busy lock (withLock throws) propagates so the caller reports skipped
*/
import { describe, it, expect } from 'bun:test';
import { readFileSync } from 'fs';
import { join } from 'path';
import {
runExtractAtomsDrain,
type ExtractAtomsDrainDeps,
} from '../src/core/cycle/extract-atoms-drain.ts';
import { isProtectedJobName, PROTECTED_JOB_NAMES } from '../src/core/minions/protected-names.ts';
function seq(values: Array<number | null>): () => Promise<number | null> {
let i = 0;
return async () => values[Math.min(i++, values.length - 1)];
}
const passThroughLock: ExtractAtomsDrainDeps['withLock'] = (work) => work();
describe('runExtractAtomsDrain (issue #1678)', () => {
it('drains to empty and reports stopped=drained', async () => {
let batches = 0;
const result = await runExtractAtomsDrain(
{
withLock: passThroughLock,
countRemaining: seq([3, 2, 1, 0, 0]),
runBatch: async () => { batches++; return { extracted: 1, skipped: 0 }; },
now: () => 0,
},
{ windowMs: 1_000_000 },
);
expect(result.stopped).toBe('drained');
expect(result.remaining).toBe(0);
expect(result.batches).toBe(3);
expect(result.extracted).toBe(3);
expect(batches).toBe(3);
});
it('stops at the wallclock window with remaining > 0', async () => {
// SYNC stepping clock: now() #1 sets deadline (0+100=100); the while-check
// then sees 50, 50 (two batches), then 999999 → past deadline → stop.
const times = [0, 50, 50, 999_999];
let ti = 0;
const now = () => times[Math.min(ti++, times.length - 1)];
const result = await runExtractAtomsDrain(
{
withLock: passThroughLock,
countRemaining: async () => 5, // never drains
runBatch: async () => ({ extracted: 1, skipped: 0 }),
now,
},
{ windowMs: 100 },
);
expect(result.stopped).toBe('window');
expect(result.remaining).toBe(5);
expect(result.batches).toBe(2);
});
it('stops on a zero-progress batch (no hot loop)', async () => {
let batches = 0;
const result = await runExtractAtomsDrain(
{
withLock: passThroughLock,
countRemaining: async () => 5,
runBatch: async () => { batches++; return { extracted: 0, skipped: 0 }; },
now: () => 0,
},
{ windowMs: 1_000_000 },
);
expect(result.stopped).toBe('no_progress');
expect(batches).toBe(1);
expect(result.remaining).toBe(5);
});
it('propagates a busy-lock error (caller reports cycle_already_running)', async () => {
class FakeBusy extends Error {}
await expect(
runExtractAtomsDrain(
{
withLock: () => { throw new FakeBusy('held'); },
countRemaining: async () => 5,
runBatch: async () => ({ extracted: 1, skipped: 0 }),
now: () => 0,
},
{ windowMs: 1000 },
),
).rejects.toThrow('held');
});
it('respects maxBatches as a belt-and-suspenders cap', async () => {
let batches = 0;
const result = await runExtractAtomsDrain(
{
withLock: passThroughLock,
countRemaining: async () => 999, // never drains
runBatch: async () => { batches++; return { extracted: 1, skipped: 0 }; },
now: () => 0, // window never elapses
},
{ windowMs: 1_000_000, maxBatches: 4 },
);
expect(result.stopped).toBe('max_batches');
expect(batches).toBe(4);
});
});
// #1685 GAP D (CODEX #1) — the auto-drain Minion job burns Haiku, so it must be
// PROTECTED: no MCP/OAuth-scoped caller can submit it; only trusted local
// callers (autopilot, explicit CLI with --allow-protected) can.
describe('extract-atoms-drain protected-name membership', () => {
it('extract-atoms-drain is PROTECTED', () => {
expect(isProtectedJobName('extract-atoms-drain')).toBe(true);
expect(PROTECTED_JOB_NAMES.has('extract-atoms-drain')).toBe(true);
});
});
// #1685 GAP D / 5A — the shared wiring helper is the single drain path. The
// "drain holds the same cycle lock id as the routine cycle" contract (moved out
// of dream.ts in the 5A refactor) lives here now.
describe('shared wiring helper holds the cycle lock (5A)', () => {
const src = readFileSync(
join(import.meta.dir, '../src/core/cycle/extract-atoms-drain.ts'),
'utf8',
);
it('runExtractAtomsDrainForSource uses cycleLockIdFor(opts.sourceId) + withRefreshingLock', () => {
expect(src).toContain('runExtractAtomsDrainForSource');
expect(src).toContain('cycleLockIdFor(opts.sourceId)');
expect(src).toContain('withRefreshingLock(engine, lockId');
});
});