mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
* feat(minions): pool-recovery audit + reconnect reason-threading + shared drain helper (#1685 GAP B, 5A) - pool-recovery-audit.ts: reap_detected (CONNECTION_ENDED) vs reconnect_other; recovered/failed split - postgres-engine reconnect(ctx?) classifies the triggering error so only true pooler reaps are tagged (CODEX #8) - retry.ts reconnect callback widened to thread the error; retry-matcher isConnectionEndedError - runExtractAtomsDrainForSource shared helper (cycleLockIdFor + withRefreshingLock) — one drain path (5A) - supervisor-audit readRecentSupervisorEvents (current+prev ISO week, CODEX #7) - extract-atoms-drain PROTECTED; autopilot.auto_drain.* config keys * feat(doctor): worker_oom_loop + pool_reap_health checks + cause-ranked top_issues (#1685 GAP A/B/C) - computeWorkerOomLoopCheck: unions supervisor rss_watchdog + minion_jobs watchdog-abort (CODEX #5), cap fallback to resolveDefaultMaxRssMb (CODEX #6) - computePoolReapHealthCheck: reaps-not-recovering fail, thrash warn - doctor-cause-rank rankIssues: tier ordering + grounded downstream_of (CODEX #9) + drift guard (4A) - supervisor causeStr + queue_health cross-reference worker_oom_loop (DRY 1C) - register both checks in doctor-categories ops * feat(autopilot): per-source extract_atoms auto-drain + handler + dream --drain refactor (#1685 GAP D) - autopilot per-source gate: enabled + !packDeclares + backlog>threshold + daily cap; time-sloted idempotency key (CODEX #2) - extract-atoms-drain Minion handler (thin wrapper, LockUnavailableError -> deferred) - dream --drain routes through the shared helper (5A) * chore: bump version and changelog (v0.42.12.0) #1685 brain-health-as-solved-problem: cause-ranked doctor, worker_oom_loop line, per-source auto-drain, pool-reap health. Layers on #1678/#1735. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * docs(todos): file #1685 GAP E + remote-path follow-ups (v0.42.12.0) * fix(#1685): pre-landing review — multi-source auto-drain, honest pool-reap signal, lock-renewal reap labeling - autopilot: drop maxWaiting (coalesces by name+queue not source → only one source drained + cap over-count); pre-check idempotency key so only genuinely-new sources submit+count - pool_reap_health: fail on reconnect FAILURES (the real signal), not reaps>0&&failures>0 (false causality when a recovered reap + unrelated failure co-occur) - lock-renewal-tick threads its triggering error to reconnect() so a CONNECTION_ENDED pooler reap is labeled reap_detected not reconnect_other (pool_reap_health now fires for the #1678 incident path) * chore: re-version v0.42.12.0 → v0.42.16.0 (#1685) Slot collision avoidance per queue. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * docs: restore slim CLAUDE.md + move #1685 entries to KEY_FILES.md (fix check:doc-history) The master merge wrongly kept the pre-restructure 577KB CLAUDE.md; the check:doc-history guard caps it at 60KB. Take master's slim CLAUDE.md and record the #1685 files (doctor-cause-rank, pool-recovery-audit, worker_oom_loop + pool_reap_health checks, auto-drain, 5A helper) as current-state prose in docs/architecture/KEY_FILES.md (no release markers). llms regenerated. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
137 lines
4.9 KiB
TypeScript
137 lines
4.9 KiB
TypeScript
/**
|
|
* issue #1678 — bounded single-hold extract_atoms drain loop.
|
|
*
|
|
* Pure-over-injected-deps, so no DB / LLM / lock primitive. Pins:
|
|
* - drains to empty (rediscovers each batch via countRemaining), stops 'drained'
|
|
* - the wallclock window bounds the loop, stops 'window' with remaining > 0
|
|
* - a zero-progress batch stops the loop (no hot loop burning budget)
|
|
* - a busy lock (withLock throws) propagates so the caller reports skipped
|
|
*/
|
|
|
|
import { describe, it, expect } from 'bun:test';
|
|
import { readFileSync } from 'fs';
|
|
import { join } from 'path';
|
|
import {
|
|
runExtractAtomsDrain,
|
|
type ExtractAtomsDrainDeps,
|
|
} from '../src/core/cycle/extract-atoms-drain.ts';
|
|
import { isProtectedJobName, PROTECTED_JOB_NAMES } from '../src/core/minions/protected-names.ts';
|
|
|
|
function seq(values: Array<number | null>): () => Promise<number | null> {
|
|
let i = 0;
|
|
return async () => values[Math.min(i++, values.length - 1)];
|
|
}
|
|
|
|
const passThroughLock: ExtractAtomsDrainDeps['withLock'] = (work) => work();
|
|
|
|
describe('runExtractAtomsDrain (issue #1678)', () => {
|
|
it('drains to empty and reports stopped=drained', async () => {
|
|
let batches = 0;
|
|
const result = await runExtractAtomsDrain(
|
|
{
|
|
withLock: passThroughLock,
|
|
countRemaining: seq([3, 2, 1, 0, 0]),
|
|
runBatch: async () => { batches++; return { extracted: 1, skipped: 0 }; },
|
|
now: () => 0,
|
|
},
|
|
{ windowMs: 1_000_000 },
|
|
);
|
|
expect(result.stopped).toBe('drained');
|
|
expect(result.remaining).toBe(0);
|
|
expect(result.batches).toBe(3);
|
|
expect(result.extracted).toBe(3);
|
|
expect(batches).toBe(3);
|
|
});
|
|
|
|
it('stops at the wallclock window with remaining > 0', async () => {
|
|
// SYNC stepping clock: now() #1 sets deadline (0+100=100); the while-check
|
|
// then sees 50, 50 (two batches), then 999999 → past deadline → stop.
|
|
const times = [0, 50, 50, 999_999];
|
|
let ti = 0;
|
|
const now = () => times[Math.min(ti++, times.length - 1)];
|
|
const result = await runExtractAtomsDrain(
|
|
{
|
|
withLock: passThroughLock,
|
|
countRemaining: async () => 5, // never drains
|
|
runBatch: async () => ({ extracted: 1, skipped: 0 }),
|
|
now,
|
|
},
|
|
{ windowMs: 100 },
|
|
);
|
|
expect(result.stopped).toBe('window');
|
|
expect(result.remaining).toBe(5);
|
|
expect(result.batches).toBe(2);
|
|
});
|
|
|
|
it('stops on a zero-progress batch (no hot loop)', async () => {
|
|
let batches = 0;
|
|
const result = await runExtractAtomsDrain(
|
|
{
|
|
withLock: passThroughLock,
|
|
countRemaining: async () => 5,
|
|
runBatch: async () => { batches++; return { extracted: 0, skipped: 0 }; },
|
|
now: () => 0,
|
|
},
|
|
{ windowMs: 1_000_000 },
|
|
);
|
|
expect(result.stopped).toBe('no_progress');
|
|
expect(batches).toBe(1);
|
|
expect(result.remaining).toBe(5);
|
|
});
|
|
|
|
it('propagates a busy-lock error (caller reports cycle_already_running)', async () => {
|
|
class FakeBusy extends Error {}
|
|
await expect(
|
|
runExtractAtomsDrain(
|
|
{
|
|
withLock: () => { throw new FakeBusy('held'); },
|
|
countRemaining: async () => 5,
|
|
runBatch: async () => ({ extracted: 1, skipped: 0 }),
|
|
now: () => 0,
|
|
},
|
|
{ windowMs: 1000 },
|
|
),
|
|
).rejects.toThrow('held');
|
|
});
|
|
|
|
it('respects maxBatches as a belt-and-suspenders cap', async () => {
|
|
let batches = 0;
|
|
const result = await runExtractAtomsDrain(
|
|
{
|
|
withLock: passThroughLock,
|
|
countRemaining: async () => 999, // never drains
|
|
runBatch: async () => { batches++; return { extracted: 1, skipped: 0 }; },
|
|
now: () => 0, // window never elapses
|
|
},
|
|
{ windowMs: 1_000_000, maxBatches: 4 },
|
|
);
|
|
expect(result.stopped).toBe('max_batches');
|
|
expect(batches).toBe(4);
|
|
});
|
|
});
|
|
|
|
// #1685 GAP D (CODEX #1) — the auto-drain Minion job burns Haiku, so it must be
|
|
// PROTECTED: no MCP/OAuth-scoped caller can submit it; only trusted local
|
|
// callers (autopilot, explicit CLI with --allow-protected) can.
|
|
describe('extract-atoms-drain protected-name membership', () => {
|
|
it('extract-atoms-drain is PROTECTED', () => {
|
|
expect(isProtectedJobName('extract-atoms-drain')).toBe(true);
|
|
expect(PROTECTED_JOB_NAMES.has('extract-atoms-drain')).toBe(true);
|
|
});
|
|
});
|
|
|
|
// #1685 GAP D / 5A — the shared wiring helper is the single drain path. The
|
|
// "drain holds the same cycle lock id as the routine cycle" contract (moved out
|
|
// of dream.ts in the 5A refactor) lives here now.
|
|
describe('shared wiring helper holds the cycle lock (5A)', () => {
|
|
const src = readFileSync(
|
|
join(import.meta.dir, '../src/core/cycle/extract-atoms-drain.ts'),
|
|
'utf8',
|
|
);
|
|
it('runExtractAtomsDrainForSource uses cycleLockIdFor(opts.sourceId) + withRefreshingLock', () => {
|
|
expect(src).toContain('runExtractAtomsDrainForSource');
|
|
expect(src).toContain('cycleLockIdFor(opts.sourceId)');
|
|
expect(src).toContain('withRefreshingLock(engine, lockId');
|
|
});
|
|
});
|