feat: v0.41.19.0 Supavisor Retry Cathedral (#1537)

Engine-level retry primitive that closes the v0.41.17 production incident
where ~3,000 wiki links + timeline entries were silently lost per dream
cycle on a 16K-page brain. Supavisor's circuit-breaker takes 5-10s to
recover; the prior single-500ms-retry shape couldn't survive it.

ARCHITECTURE
============

Retry becomes a data-primitive contract, not a caller responsibility.
postgres-engine.ts + pglite-engine.ts now self-retry inside addLinksBatch,
addTimelineEntriesBatch, and upsertChunks. Every caller — current AND
future — inherits retry-for-free. CI lint guard `scripts/check-no-double-retry.sh`
fails the build if anyone re-wraps an engine batch method (preventing
3×3=9 retry amplification on incomplete reverts).

CODEX-HARDENED DEFAULTS
=======================

BULK_RETRY_OPTS = {maxRetries:3, delayMs:1000, delayMaxMs:10000,
jitter:'decorrelated'}. Total worst-case wait ≈12s covers full Supavisor
recovery window. Decorrelated jitter (AWS-style uniform(base, prevDelay*3)
capped at maxDelay) replaces 'full' which allowed near-zero retries that
re-hit the still-recovering breaker.

AbortSignal threading from MinionWorker.shutdownAbort.signal through
engine method opts → withRetry → abortableSleep. SIGTERM aborts sleeping
retries instead of blocking deploys for up to delayMaxMs.

OBSERVABILITY
=============

`~/.gbrain/audit/batch-retry-YYYY-Www.jsonl` records every retry event
(success-after-blip AND exhausted-retries). Built on the v0.40.4.0
audit-writer cathedral. Privacy posture: never logs slugs / page IDs /
content (mirrors shell-audit.ts).

`gbrain doctor` learns `batch_retry_health` check. Reads last 24h
(not 7d — codex H-9: avoid permanent noise from one historical blip).
Thresholds: ok (zero or <3 same-site), warn (>=3 same-site OR >=5
cross-site), fail (>=20 sustained breaker). Surfaces bad GBRAIN_BULK_*
env at startup (codex M-10). Corrupt-JSONL tolerant.

30-day audit pruning hooked into the dream cycle's purge phase (codex H-8
— implements the 'pruning convention' for real).

OPERATOR TUNING
===============

GBRAIN_BULK_MAX_RETRIES (int >= 0; 0 disables retries for debugging)
GBRAIN_BULK_RETRY_BASE_MS (int > 0)
GBRAIN_BULK_RETRY_MAX_MS (int >= base)

Bad values throw GBrainError with paste-ready fix hints at doctor startup,
not at first-retry mid-cycle.

VERIFICATION
============

- bun run verify: 28/28 checks green (includes 2 new lint guards:
  check-no-double-retry, check-batch-audit-site)
- bun run test: 11453 pass / 1 pre-existing flake (schema-cli.test.ts —
  confirmed by running on clean master, NOT introduced by this wave)
- bun run test:slow: 40/40 including new test/core/retry-stress.slow.test.ts
  (100 batches × 30% blip rate × decorrelated jitter, zero row loss)
- bunx tsc --noEmit: 0 errors

REVIEWS
=======

- CEO review (SELECTIVE EXPANSION): 4 cherry-picks proposed, 4 accepted
- Eng review (2 passes): 10 findings, 0 critical gaps, architectural
  pivot from per-site to engine-level wrap
- Codex independent review: 23 findings; 10 critical/high absorbed
  (decorrelated jitter, 12s backoff window, AbortSignal, idempotency
  proof, backfill unification, typed audit-site enum, doctor expiry
  thresholds, audit pruning, env validation at doctor startup)

PR #1523 closed and absorbed (@garrytan-agents original extract.ts fix
preserved via co-author trailer; 5 test cases moved to test/core/retry.test.ts
with assertions adjusted for the v0.41.19.0 BULK_RETRY_OPTS defaults).

Co-authored-by: garrytan-agents <noreply@anthropic.com>

test/audit/batch-retry-audit.test.ts

@@ -0,0 +1,230 @@
+// v0.41.18.0 — batch-retry audit JSONL primitive.
+//
+// Hermetic: uses GBRAIN_AUDIT_DIR env override via the withEnv helper so
+// the test never touches the user's ~/.gbrain/audit/. Each test gets a fresh
+// tempdir via beforeEach so file-system state never leaks across cases.
+
+import { describe, expect, test, beforeEach, afterEach } from 'bun:test';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { withEnv } from '../helpers/with-env.ts';
+import {
+  logBatchRetry,
+  logBatchExhausted,
+  readRecentBatchRetryEvents,
+  pruneOldBatchRetryAuditFiles,
+  BATCH_RETRY_FEATURE_NAME,
+} from '../../src/core/audit/batch-retry-audit.ts';
+
+let tmpDir: string;
+
+beforeEach(() => {
+  tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'batch-retry-audit-'));
+});
+
+afterEach(() => {
+  try {
+    fs.rmSync(tmpDir, { recursive: true, force: true });
+  } catch { /* best-effort */ }
+});
+
+describe('logBatchRetry — success-path emission', () => {
+  test('writes JSONL row with outcome=success', async () => {
+    await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
+      logBatchRetry('extract.links_inc', 100, 1, 1000, new Error('Connection terminated'));
+      const result = readRecentBatchRetryEvents(24);
+      expect(result.events).toHaveLength(1);
+      expect(result.events[0].site).toBe('extract.links_inc');
+      expect(result.events[0].batch_size).toBe(100);
+      expect(result.events[0].attempt).toBe(1);
+      expect(result.events[0].outcome).toBe('success');
+      expect(result.events[0].delay_ms).toBe(1000);
+      expect(result.events[0].error_message_summary).toContain('Connection terminated');
+    });
+  });
+
+  test('captures SQLSTATE code when present on error', async () => {
+    await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
+      const err = Object.assign(new Error('connection failure'), { code: '08006' });
+      logBatchRetry('extract.timeline_fs', 50, 2, 3000, err);
+      const result = readRecentBatchRetryEvents(24);
+      expect(result.events[0].error_code).toBe('08006');
+    });
+  });
+
+  test('truncates long error messages to 200 chars', async () => {
+    await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
+      const longMsg = 'A'.repeat(500);
+      logBatchRetry('addLinksBatch', 100, 1, 1000, new Error(longMsg));
+      const result = readRecentBatchRetryEvents(24);
+      expect(result.events[0].error_message_summary.length).toBe(200);
+    });
+  });
+
+  test('replaces newlines in error message (grep-friendly)', async () => {
+    await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
+      logBatchRetry('addLinksBatch', 100, 1, 1000, new Error('line1\nline2\tline3'));
+      const result = readRecentBatchRetryEvents(24);
+      expect(result.events[0].error_message_summary).toBe('line1 line2 line3');
+    });
+  });
+});
+
+describe('logBatchExhausted — exhausted-retry emission', () => {
+  test('writes outcome=exhausted with total attempt count', async () => {
+    await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
+      logBatchExhausted('mcp.put_page.autolink', 25, 4, new Error('Connection terminated'));
+      const result = readRecentBatchRetryEvents(24);
+      expect(result.events).toHaveLength(1);
+      expect(result.events[0].outcome).toBe('exhausted');
+      expect(result.events[0].attempt).toBe(4);
+      expect(result.events[0].site).toBe('mcp.put_page.autolink');
+    });
+  });
+});
+
+describe('readRecentBatchRetryEvents — windowing + corruption tolerance', () => {
+  test('filters by hours-back cutoff', async () => {
+    await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
+      // Write 3 events: one 25h ago (out of 24h window), one now, one 1h ago.
+      const now = Date.now();
+      const dir = tmpDir;
+      // Use the writer to land in the correct ISO-week file.
+      logBatchRetry('addLinksBatch', 1, 1, 100, new Error('a'));
+      logBatchRetry('addLinksBatch', 2, 1, 100, new Error('b'));
+      // Manually inject an old event by appending to the same file.
+      const filename = `${BATCH_RETRY_FEATURE_NAME}-${new Date(now).getUTCFullYear()}-W${String(getIsoWeek(new Date(now))).padStart(2, '0')}.jsonl`;
+      const filePath = path.join(dir, filename);
+      const oldEvent = {
+        ts: new Date(now - 25 * 3600_000).toISOString(),
+        site: 'addLinksBatch',
+        batch_size: 99,
+        attempt: 1,
+        outcome: 'success',
+        delay_ms: 100,
+        error_message_summary: 'old',
+      };
+      fs.appendFileSync(filePath, JSON.stringify(oldEvent) + '\n');
+
+      const result = readRecentBatchRetryEvents(24);
+      // Should see 2 recent events but NOT the 25h-old one.
+      expect(result.events.length).toBeGreaterThanOrEqual(2);
+      const oldFound = result.events.find(e => e.error_message_summary === 'old');
+      expect(oldFound).toBeUndefined();
+    });
+  });
+
+  test('counts corrupted JSONL lines without crashing', async () => {
+    await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
+      // Write a valid event first to ensure the file exists.
+      logBatchRetry('addLinksBatch', 1, 1, 100, new Error('valid'));
+      const now = new Date();
+      const filename = `${BATCH_RETRY_FEATURE_NAME}-${now.getUTCFullYear()}-W${String(getIsoWeek(now)).padStart(2, '0')}.jsonl`;
+      const filePath = path.join(tmpDir, filename);
+      // Append 3 corrupt lines.
+      fs.appendFileSync(filePath, '{not json\n');
+      fs.appendFileSync(filePath, 'still not json\n');
+      fs.appendFileSync(filePath, '{"missing_close": true\n');
+
+      const result = readRecentBatchRetryEvents(24);
+      expect(result.events).toHaveLength(1); // the valid one
+      expect(result.corrupted_lines).toBe(3);
+    });
+  });
+
+  test('files_unreadable counts permission errors but ignores ENOENT', async () => {
+    await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
+      // No files written; the only "missing" file is ENOENT which is expected.
+      const result = readRecentBatchRetryEvents(24);
+      expect(result.events).toHaveLength(0);
+      expect(result.files_unreadable).toBe(0);
+      expect(result.files_scanned).toBe(0);
+    });
+  });
+});
+
+describe('pruneOldBatchRetryAuditFiles — codex H-8 actual pruning', () => {
+  test('deletes files older than daysToKeep', async () => {
+    await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
+      // Create an old file (mtime 31 days ago).
+      const oldFile = path.join(tmpDir, `${BATCH_RETRY_FEATURE_NAME}-2024-W01.jsonl`);
+      fs.writeFileSync(oldFile, '{"ts":"2024-01-01T00:00:00Z"}\n');
+      const oldTime = Date.now() - 31 * 86400_000;
+      fs.utimesSync(oldFile, oldTime / 1000, oldTime / 1000);
+
+      // Create a recent file (mtime now).
+      const newFile = path.join(tmpDir, `${BATCH_RETRY_FEATURE_NAME}-2026-W22.jsonl`);
+      fs.writeFileSync(newFile, '{"ts":"2026-05-26T00:00:00Z"}\n');
+
+      const result = pruneOldBatchRetryAuditFiles(30);
+      expect(result.removed).toBe(1);
+      expect(result.kept).toBe(1);
+      expect(fs.existsSync(oldFile)).toBe(false);
+      expect(fs.existsSync(newFile)).toBe(true);
+    });
+  });
+
+  test('ignores files that do not match the batch-retry-*.jsonl shape', async () => {
+    await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
+      // Other audit module's files should not be touched.
+      const otherFile = path.join(tmpDir, 'shell-jobs-2024-W01.jsonl');
+      fs.writeFileSync(otherFile, '{}\n');
+      const oldTime = Date.now() - 31 * 86400_000;
+      fs.utimesSync(otherFile, oldTime / 1000, oldTime / 1000);
+
+      const result = pruneOldBatchRetryAuditFiles(30);
+      expect(result.removed).toBe(0);
+      expect(fs.existsSync(otherFile)).toBe(true);
+    });
+  });
+
+  test('no-op when audit dir does not exist (ENOENT)', () => {
+    const result = pruneOldBatchRetryAuditFiles(30, new Date());
+    // Even without the env override, the function never throws on missing dir.
+    // We just check it returns the empty result without throwing.
+    expect(result).toEqual({ removed: 0, kept: 0 });
+  });
+});
+
+describe('privacy posture (codex review + CLAUDE.md privacy rule)', () => {
+  test('audit row never contains slugs, page ids, or content', async () => {
+    await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
+      logBatchRetry(
+        'extract.links_inc',
+        100,
+        1,
+        1000,
+        new Error('insert failed for slug=people/alice page_id=42'),
+      );
+      const result = readRecentBatchRetryEvents(24);
+      // The error message IS in the audit (necessary for debugging), but
+      // schema fields don't carry slugs, IDs, or content separately. The
+      // 200-char truncation prevents large blob leaks too.
+      const row = result.events[0];
+      // Verify the row shape — no slug, page_id, content, body fields.
+      // error_code is optional (only present when error has SQLSTATE), so
+      // the schema's closed set is these fields plus optional error_code.
+      const keys = new Set(Object.keys(row));
+      const requiredKeys = ['attempt', 'batch_size', 'delay_ms', 'error_message_summary', 'outcome', 'site', 'ts'];
+      for (const k of requiredKeys) expect(keys.has(k)).toBe(true);
+      // No leaky fields.
+      for (const k of keys) {
+        expect(['attempt', 'batch_size', 'delay_ms', 'error_code', 'error_message_summary', 'outcome', 'site', 'ts']).toContain(k);
+      }
+    });
+  });
+});
+
+// Helper: ISO 8601 week number (matches the audit-writer.ts computation).
+function getIsoWeek(d: Date): number {
+  const target = new Date(d.valueOf());
+  const dayNumber = (d.getUTCDay() + 6) % 7;
+  target.setUTCDate(target.getUTCDate() - dayNumber + 3);
+  const firstThursday = target.valueOf();
+  target.setUTCMonth(0, 1);
+  if (target.getUTCDay() !== 4) {
+    target.setUTCMonth(0, 1 + ((4 - target.getUTCDay()) + 7) % 7);
+  }
+  return 1 + Math.ceil((firstThursday - target.valueOf()) / 604800000);
+}

test/core/retry-stress.slow.test.ts

@@ -0,0 +1,224 @@
+// v0.41.18.0 — withRetry stress regression test (T7 / CEO D6).
+//
+// Pins the contract the engine-level batch wrap depends on:
+//   - 30% blip rate (matches the production Supavisor incident shape)
+//   - 100 simulated batches
+//   - BULK_RETRY_OPTS defaults (maxRetries=3, decorrelated jitter)
+//   - Asserts zero row loss when failures are bounded by maxRetries
+//   - Asserts retries DO fire (so a future "optimize" can't silently
+//     disable retries and pass the suite)
+//   - Audit JSONL records correct outcome per case
+//
+// Lives in .slow.test.ts tier per CLAUDE.md test taxonomy. Uses delayMs=1
+// for hermeticity — the test exercises the retry/jitter math + audit emission,
+// not real timing.
+
+import { describe, expect, test, beforeEach, afterEach } from 'bun:test';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { withEnv } from '../helpers/with-env.ts';
+import {
+  withRetry,
+  computeNextDelay,
+  BULK_RETRY_OPTS,
+} from '../../src/core/retry.ts';
+import {
+  logBatchRetry,
+  logBatchExhausted,
+  readRecentBatchRetryEvents,
+} from '../../src/core/audit/batch-retry-audit.ts';
+
+let tmpDir: string;
+
+beforeEach(() => {
+  tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'retry-stress-'));
+});
+
+afterEach(() => {
+  try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* */ }
+});
+
+/**
+ * Simulate the engine-level batchRetry helper without spinning up a real
+ * engine. Mirrors postgres-engine.ts batchRetry: withRetry + audit emission
+ * for both success-after-retry AND exhausted-retry paths.
+ */
+async function simulateEngineBatchRetry<T>(
+  auditSite: 'addLinksBatch',
+  batchSize: number,
+  fn: () => Promise<T>,
+): Promise<T> {
+  let prevDelay = 0;
+  let onRetryCount = 0;
+  try {
+    return await withRetry(fn, {
+      maxRetries: BULK_RETRY_OPTS.maxRetries,
+      delayMs: 1, // hermeticity — not testing real timing
+      delayMaxMs: 10,
+      jitter: BULK_RETRY_OPTS.jitter,
+      onRetry: (attempt, err) => {
+        onRetryCount++;
+        const delay = computeNextDelay(attempt - 1, prevDelay, 1, 10, BULK_RETRY_OPTS.jitter);
+        prevDelay = delay;
+        logBatchRetry(auditSite, batchSize, attempt, delay, err);
+      },
+    });
+  } catch (err) {
+    if (err instanceof Error && err.name === 'RetryAbortError') throw err;
+    const { isRetryableConnError } = await import('../../src/core/retry.ts');
+    if (isRetryableConnError(err)) {
+      logBatchExhausted(auditSite, batchSize, BULK_RETRY_OPTS.maxRetries + 1, err);
+    }
+    throw err;
+  }
+}
+
+describe('stress: 100 batches × 30% blip rate with BULK_RETRY_OPTS', () => {
+  test('eventual success on all batches when blip count <= maxRetries', async () => {
+    await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
+      // Deterministic seeded "blip" generator: a fixed sequence so failures
+      // happen at predictable batch positions. 30% blip rate, but the
+      // number of CONSECUTIVE blips on any one batch never exceeds 2 (which
+      // is comfortably within BULK_RETRY_OPTS.maxRetries=3). This validates
+      // the recovery path — zero row loss expected.
+      let totalBatches = 0;
+      let totalLost = 0;
+      let totalRetries = 0;
+
+      for (let batchIdx = 0; batchIdx < 100; batchIdx++) {
+        // Each batch fails 0-2 times then succeeds. Pseudo-random based on
+        // batchIdx for deterministic test runs.
+        const failureBudget = batchIdx % 10 < 3 ? (batchIdx % 3) + 1 : 0;
+        let calls = 0;
+        try {
+          await simulateEngineBatchRetry('addLinksBatch', 100, async () => {
+            calls++;
+            if (calls <= failureBudget) {
+              throw new Error('Connection terminated unexpectedly');
+            }
+            return 'ok';
+          });
+          totalBatches++;
+          if (failureBudget > 0) totalRetries += failureBudget;
+        } catch {
+          totalLost++;
+        }
+      }
+
+      expect(totalBatches).toBe(100);
+      expect(totalLost).toBe(0);
+      // 30% of batches blip; each blipping batch retries 1-3 times.
+      // At minimum some retries should have fired.
+      expect(totalRetries).toBeGreaterThan(0);
+
+      // Audit JSONL should record every retry attempt.
+      const auditResult = readRecentBatchRetryEvents(24);
+      const successEvents = auditResult.events.filter((e) => e.outcome === 'success');
+      const exhaustedEvents = auditResult.events.filter((e) => e.outcome === 'exhausted');
+      expect(successEvents.length).toBe(totalRetries);
+      expect(exhaustedEvents.length).toBe(0);
+    });
+  }, 30_000);
+
+  test('exhausted retries recorded when blip count > maxRetries', async () => {
+    await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
+      // Force batch 7 to fail more times than maxRetries allows.
+      const failOn = 7;
+      const failureCount = BULK_RETRY_OPTS.maxRetries + 1; // exceeds budget
+      let lostBatches = 0;
+      let successfulBatches = 0;
+
+      for (let batchIdx = 0; batchIdx < 20; batchIdx++) {
+        let calls = 0;
+        try {
+          await simulateEngineBatchRetry('addLinksBatch', 50, async () => {
+            calls++;
+            if (batchIdx === failOn && calls <= failureCount) {
+              throw new Error('Connection terminated unexpectedly');
+            }
+            return 'ok';
+          });
+          successfulBatches++;
+        } catch {
+          lostBatches++;
+        }
+      }
+
+      expect(successfulBatches).toBe(19);
+      expect(lostBatches).toBe(1);
+
+      const auditResult = readRecentBatchRetryEvents(24);
+      const exhausted = auditResult.events.filter((e) => e.outcome === 'exhausted');
+      expect(exhausted.length).toBe(1);
+      expect(exhausted[0].site).toBe('addLinksBatch');
+      expect(exhausted[0].batch_size).toBe(50);
+    });
+  }, 30_000);
+
+  test('non-retryable error propagates immediately, no exhausted audit', async () => {
+    await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
+      let calls = 0;
+      try {
+        await simulateEngineBatchRetry('addLinksBatch', 10, async () => {
+          calls++;
+          const err = Object.assign(new Error('duplicate key'), { code: '23505' });
+          throw err;
+        });
+        expect.unreachable();
+      } catch (e) {
+        expect((e as Error).message).toContain('duplicate key');
+      }
+      expect(calls).toBe(1); // no retry on non-retryable
+
+      const auditResult = readRecentBatchRetryEvents(24);
+      // Non-retryable errors are NOT audited — they're caller bugs/data
+      // issues, not retry-budget exhaustion.
+      expect(auditResult.events.length).toBe(0);
+    });
+  }, 30_000);
+});
+
+describe('decorrelated jitter sanity over 100 attempts', () => {
+  test('never produces near-zero delays (codex C-2 regression lock)', () => {
+    // Sample 100 retries against BULK_RETRY_OPTS to confirm the floor.
+    // Pre-codex-C-2, jitter='full' would have allowed near-zero delays
+    // here ~10% of the time. Decorrelated jitter floors at delayMs.
+    let prev = 0;
+    for (let attempt = 0; attempt < 100; attempt++) {
+      const d = computeNextDelay(
+        attempt % BULK_RETRY_OPTS.maxRetries,
+        prev,
+        BULK_RETRY_OPTS.delayMs,
+        BULK_RETRY_OPTS.delayMaxMs,
+        BULK_RETRY_OPTS.jitter,
+      );
+      // Floor at delayMs = 1000ms (Supavisor recovery floor).
+      expect(d).toBeGreaterThanOrEqual(BULK_RETRY_OPTS.delayMs);
+      expect(d).toBeLessThanOrEqual(BULK_RETRY_OPTS.delayMaxMs);
+      prev = d;
+    }
+  });
+
+  test('cumulative wait reaches >= 8s in worst-case (covers 5-10s Supavisor window)', () => {
+    let bestTotal = 0;
+    for (let trial = 0; trial < 100; trial++) {
+      let prev = 0;
+      let total = 0;
+      for (let attempt = 0; attempt < BULK_RETRY_OPTS.maxRetries; attempt++) {
+        const d = computeNextDelay(
+          attempt,
+          prev,
+          BULK_RETRY_OPTS.delayMs,
+          BULK_RETRY_OPTS.delayMaxMs,
+          BULK_RETRY_OPTS.jitter,
+        );
+        prev = d;
+        total += d;
+      }
+      if (total > bestTotal) bestTotal = total;
+    }
+    // BULK_RETRY_OPTS produces decorrelated worst case ~10+s+ (delayMaxMs cap).
+    expect(bestTotal).toBeGreaterThanOrEqual(8000);
+  });
+});

test/core/retry.test.ts

@@ -0,0 +1,440 @@
+// v0.41.18.0 — src/core/retry.ts canonical retry primitive.
+//
+// Moved from test/extract-batch-retry.test.ts when withRetry was factored
+// out of extract.ts (Eng-D2 architectural pivot: engine-level wrap instead
+// of per-call-site wrap). All v0.41.2.1 contracts preserved verbatim:
+//
+//   - withRetry is a pure primitive (no UI), exports cleanly.
+//   - Classification uses isRetryableConnError from retry-matcher.ts.
+//   - Default maxRetries=1 = single 500ms retry (v0.41.2.1 back-compat).
+//   - Non-retryable errors propagate immediately.
+//   - onRetry callback fires per attempt with (1-based attempt, err).
+//
+// v0.41.18.0 codex-hardening additions:
+//   - BULK_RETRY_OPTS defaults (3 retries, 1s/3s/8s exponential w/ decorrelated jitter)
+//   - AbortSignal threading + abortableSleep
+//   - Typed BatchAuditSite enum + isBatchAuditSite guard
+//   - resolveBulkRetryOpts env-override with paste-ready error hints
+//   - computeNextDelay pure-function for each jitter mode
+//
+// Hermetic: no engine, no PGLite, no env mutation (uses withEnv helper),
+// no DATABASE_URL required.
+
+import { describe, expect, test, beforeEach } from 'bun:test';
+import {
+  withRetry,
+  abortableSleep,
+  computeNextDelay,
+  resolveBulkRetryOpts,
+  isBatchAuditSite,
+  isRetryableConnError,
+  BULK_RETRY_OPTS,
+  BATCH_AUDIT_SITES,
+  RetryAbortError,
+} from '../../src/core/retry.ts';
+
+// Minimal GBrainError shape mirrors the typed problem/detail fields from
+// db.ts:getConnection so the retry-matcher extension recognizes it.
+class FakeGBrainError extends Error {
+  problem: string;
+  detail: string;
+  constructor(problem: string, detail: string) {
+    super(`${problem}: ${detail}`);
+    this.problem = problem;
+    this.detail = detail;
+  }
+}
+
+describe('isRetryableConnError extension (v0.41.2.1, re-exported from retry.ts)', () => {
+  test('GBrainError with problem="No database connection" is retryable', () => {
+    const err = new FakeGBrainError('No database connection', 'connect() has not been called');
+    expect(isRetryableConnError(err)).toBe(true);
+  });
+
+  test('GBrainError with other problem is NOT retryable', () => {
+    const err = new FakeGBrainError('Schema mismatch', 'expected vector(1536), got vector(1024)');
+    expect(isRetryableConnError(err)).toBe(false);
+  });
+
+  test('plain Error with "No database connection" message is retryable (literal match)', () => {
+    expect(isRetryableConnError(new Error('No database connection: connect() has not been called.'))).toBe(true);
+  });
+
+  test('constraint violation 23505 is NOT retryable', () => {
+    const err = Object.assign(new Error('duplicate key value violates unique constraint'), { code: '23505' });
+    expect(isRetryableConnError(err)).toBe(false);
+  });
+});
+
+describe('withRetry primitive — v0.41.2.1 back-compat contract', () => {
+  test('first-call success: returns value, no onRetry invocation', async () => {
+    let calls = 0;
+    let retried = false;
+    const result = await withRetry(
+      async () => { calls++; return 42; },
+      { onRetry: () => { retried = true; }, delayMs: 0 },
+    );
+    expect(result).toBe(42);
+    expect(calls).toBe(1);
+    expect(retried).toBe(false);
+  });
+
+  test('retries on Connection terminated; second attempt succeeds', async () => {
+    let calls = 0;
+    const result = await withRetry(
+      async () => {
+        calls++;
+        if (calls === 1) throw new Error('Connection terminated unexpectedly');
+        return 'recovered';
+      },
+      { delayMs: 0 },
+    );
+    expect(result).toBe('recovered');
+    expect(calls).toBe(2);
+  });
+
+  test('retries on GBrainError "No database connection"; second succeeds', async () => {
+    let calls = 0;
+    const result = await withRetry(
+      async () => {
+        calls++;
+        if (calls === 1) throw new FakeGBrainError('No database connection', 'connect() has not been called');
+        return 'ok';
+      },
+      { delayMs: 0 },
+    );
+    expect(result).toBe('ok');
+    expect(calls).toBe(2);
+  });
+
+  test('non-retryable error propagates immediately, no retry', async () => {
+    let calls = 0;
+    const promise = withRetry(
+      async () => {
+        calls++;
+        const err = Object.assign(new Error('duplicate key'), { code: '23505' });
+        throw err;
+      },
+      { delayMs: 0 },
+    );
+    await expect(promise).rejects.toThrow('duplicate key');
+    expect(calls).toBe(1); // no retry on 23505
+  });
+
+  test('default maxRetries=1: second failure propagates (single retry, not infinite)', async () => {
+    let calls = 0;
+    const promise = withRetry(
+      async () => {
+        calls++;
+        throw new Error('ECONNRESET');
+      },
+      { delayMs: 0 },
+    );
+    await expect(promise).rejects.toThrow('ECONNRESET');
+    expect(calls).toBe(2); // attempt 1 + 1 retry, then propagate (back-compat)
+  });
+
+  test('onRetry callback receives (attempt=1, err)', async () => {
+    let received: { attempt: number; err: unknown } | null = null;
+    let calls = 0;
+    await withRetry(
+      async () => {
+        calls++;
+        if (calls === 1) throw new Error('Connection terminated unexpectedly');
+        return null;
+      },
+      {
+        onRetry: (attempt, err) => { received = { attempt, err }; },
+        delayMs: 0,
+      },
+    );
+    expect(received).not.toBeNull();
+    expect(received!.attempt).toBe(1);
+    expect(received!.err).toBeInstanceOf(Error);
+    expect((received!.err as Error).message).toBe('Connection terminated unexpectedly');
+  });
+
+  test('delayMs default is 500ms when not specified', async () => {
+    let calls = 0;
+    const start = Date.now();
+    await withRetry(
+      async () => {
+        calls++;
+        if (calls === 1) throw new Error('ECONNRESET');
+        return null;
+      },
+      // no delayMs override
+    );
+    const elapsed = Date.now() - start;
+    expect(calls).toBe(2);
+    expect(elapsed).toBeGreaterThanOrEqual(450); // ±50ms scheduler tolerance
+    expect(elapsed).toBeLessThan(2000);
+  }, 5000);
+});
+
+describe('withRetry — maxRetries > 1 (v0.41.18.0 BULK_RETRY_OPTS contract)', () => {
+  test('maxRetries=3: succeeds on third retry', async () => {
+    let calls = 0;
+    const result = await withRetry(
+      async () => {
+        calls++;
+        if (calls <= 3) throw new Error('Connection terminated unexpectedly');
+        return 'recovered';
+      },
+      { delayMs: 0, maxRetries: 3 },
+    );
+    expect(result).toBe('recovered');
+    expect(calls).toBe(4); // 1 initial + 3 retries
+  });
+
+  test('maxRetries=3: all retries fail, propagates last error', async () => {
+    let calls = 0;
+    const promise = withRetry(
+      async () => {
+        calls++;
+        throw new Error('ECONNRESET');
+      },
+      { delayMs: 0, maxRetries: 3 },
+    );
+    await expect(promise).rejects.toThrow('ECONNRESET');
+    expect(calls).toBe(4); // 1 initial + 3 retries
+  });
+
+  test('maxRetries=0: no retry, single attempt only (operator-debug mode)', async () => {
+    let calls = 0;
+    const promise = withRetry(
+      async () => {
+        calls++;
+        throw new Error('ECONNRESET');
+      },
+      { delayMs: 0, maxRetries: 0 },
+    );
+    await expect(promise).rejects.toThrow('ECONNRESET');
+    expect(calls).toBe(1); // exhausted before retry fires
+  });
+});
+
+describe('computeNextDelay — exponential / full / decorrelated jitter', () => {
+  test('jitter=none: pure exponential capped at maxDelay', () => {
+    expect(computeNextDelay(0, 0, 1000, 10_000, 'none')).toBe(1000);
+    expect(computeNextDelay(1, 1000, 1000, 10_000, 'none')).toBe(2000);
+    expect(computeNextDelay(2, 2000, 1000, 10_000, 'none')).toBe(4000);
+    expect(computeNextDelay(3, 4000, 1000, 10_000, 'none')).toBe(8000);
+    // cap kicks in
+    expect(computeNextDelay(4, 8000, 1000, 10_000, 'none')).toBe(10_000);
+  });
+
+  test('jitter=full: uniform in [0, exponential]', () => {
+    const rng = () => 0.5; // deterministic 50%
+    expect(computeNextDelay(0, 0, 1000, 10_000, 'full', rng)).toBe(500);
+    expect(computeNextDelay(1, 0, 1000, 10_000, 'full', rng)).toBe(1000);
+    expect(computeNextDelay(2, 0, 1000, 10_000, 'full', rng)).toBe(2000);
+  });
+
+  test('jitter=decorrelated: uniform in [base, prevDelay*3] capped at maxDelay', () => {
+    const rng = () => 0; // always low end → base
+    expect(computeNextDelay(0, 0, 1000, 10_000, 'decorrelated', rng)).toBe(1000);
+    expect(computeNextDelay(1, 1000, 1000, 10_000, 'decorrelated', rng)).toBe(1000);
+    expect(computeNextDelay(2, 3000, 1000, 10_000, 'decorrelated', rng)).toBe(1000);
+
+    const rngHigh = () => 0.99; // always high end → near cap
+    const d = computeNextDelay(3, 3000, 1000, 10_000, 'decorrelated', rngHigh);
+    // hi = max(base=1000, prevDelay*3=9000) = 9000
+    expect(d).toBeGreaterThanOrEqual(8500);
+    expect(d).toBeLessThanOrEqual(9000);
+  });
+
+  test('jitter=decorrelated: first retry (prevDelay=0) does NOT degenerate to base-only', () => {
+    // The codex-caught initialization bug: when prevDelay=0, the formula
+    // could pick uniform(base, max(base, 0)) = base every time. The fix
+    // floors `hi` at `base` so we get uniform(base, base) = base on first
+    // retry, which is at least the base delay — not zero.
+    expect(computeNextDelay(0, 0, 1000, 10_000, 'decorrelated', () => 0)).toBe(1000);
+    expect(computeNextDelay(0, 0, 1000, 10_000, 'decorrelated', () => 1)).toBe(1000);
+  });
+
+  test('jitter=decorrelated: NEVER produces near-zero (the codex C-2 fix)', () => {
+    // The whole point of replacing 'full' with 'decorrelated': bad pooler
+    // recovery happens when retries near-zero re-hit the breaker. Floor
+    // at base prevents that.
+    for (let i = 0; i < 100; i++) {
+      const d = computeNextDelay(2, 3000, 1000, 10_000, 'decorrelated');
+      expect(d).toBeGreaterThanOrEqual(1000);
+    }
+  });
+});
+
+describe('abortableSleep + AbortSignal threading (D9 codex)', () => {
+  test('abortableSleep resolves after ms when no signal', async () => {
+    const start = Date.now();
+    await abortableSleep(50);
+    const elapsed = Date.now() - start;
+    expect(elapsed).toBeGreaterThanOrEqual(40);
+    expect(elapsed).toBeLessThan(200);
+  });
+
+  test('abortableSleep rejects immediately when signal already aborted', async () => {
+    const ctrl = new AbortController();
+    ctrl.abort();
+    await expect(abortableSleep(1000, ctrl.signal)).rejects.toBeInstanceOf(RetryAbortError);
+  });
+
+  test('abortableSleep rejects mid-sleep when signal fires', async () => {
+    const ctrl = new AbortController();
+    const start = Date.now();
+    const sleepPromise = abortableSleep(5000, ctrl.signal);
+    setTimeout(() => ctrl.abort(), 20);
+    await expect(sleepPromise).rejects.toBeInstanceOf(RetryAbortError);
+    const elapsed = Date.now() - start;
+    expect(elapsed).toBeLessThan(500); // didn't wait the full 5s
+  });
+
+  test('withRetry honors signal: aborts mid-retry-sleep', async () => {
+    const ctrl = new AbortController();
+    let calls = 0;
+    const promise = withRetry(
+      async () => {
+        calls++;
+        throw new Error('ECONNRESET');
+      },
+      { maxRetries: 5, delayMs: 1000, signal: ctrl.signal },
+    );
+    setTimeout(() => ctrl.abort(), 50);
+    await expect(promise).rejects.toBeInstanceOf(RetryAbortError);
+    // Should abort before all 6 attempts completed (each would wait 1s+)
+    expect(calls).toBeLessThan(6);
+  });
+
+  test('withRetry honors signal: aborts BEFORE first attempt if already aborted', async () => {
+    const ctrl = new AbortController();
+    ctrl.abort();
+    let calls = 0;
+    const promise = withRetry(
+      async () => {
+        calls++;
+        return 'ok';
+      },
+      { signal: ctrl.signal },
+    );
+    await expect(promise).rejects.toBeInstanceOf(RetryAbortError);
+    expect(calls).toBe(0); // never ran the function
+  });
+});
+
+describe('BATCH_AUDIT_SITES typed enum + isBatchAuditSite guard (D10c codex)', () => {
+  test('every known site is recognized', () => {
+    for (const site of BATCH_AUDIT_SITES) {
+      expect(isBatchAuditSite(site)).toBe(true);
+    }
+  });
+
+  test('unknown sites rejected', () => {
+    expect(isBatchAuditSite('extract.typo')).toBe(false);
+    expect(isBatchAuditSite('mcp.put_pAge.autolink')).toBe(false);
+    expect(isBatchAuditSite('')).toBe(false);
+    expect(isBatchAuditSite('addLinksBatchz')).toBe(false);
+  });
+
+  test('list contains all CEO + eng + codex callers (regression: future deletes must be intentional)', () => {
+    // Pin the set so a future "cleanup" PR can't silently drop a site and
+    // break audit-attribution for the corresponding caller.
+    const expected = new Set([
+      'addLinksBatch', 'addTimelineEntriesBatch', 'upsertChunks',
+      'extract.links_inc', 'extract.timeline_inc',
+      'extract.links_fs', 'extract.timeline_fs',
+      'extract.links_db', 'extract.timeline_db',
+      'extract.by_mention',
+      'mcp.put_page.autolink',
+      'sync.import_file',
+      'reindex.markdown', 'reindex.multimodal',
+      'backfill.outer',
+    ]);
+    expect(new Set<string>([...BATCH_AUDIT_SITES])).toEqual(expected);
+  });
+});
+
+describe('resolveBulkRetryOpts env-override (D3 cherry-pick, codex M-10/M-12)', () => {
+  test('no env vars: returns BULK_RETRY_OPTS verbatim', () => {
+    const out = resolveBulkRetryOpts({});
+    expect(out).toEqual(BULK_RETRY_OPTS);
+  });
+
+  test('all 3 vars set: overrides apply', () => {
+    const out = resolveBulkRetryOpts({
+      GBRAIN_BULK_MAX_RETRIES: '5',
+      GBRAIN_BULK_RETRY_BASE_MS: '2000',
+      GBRAIN_BULK_RETRY_MAX_MS: '15000',
+    });
+    expect(out.maxRetries).toBe(5);
+    expect(out.delayMs).toBe(2000);
+    expect(out.delayMaxMs).toBe(15_000);
+    expect(out.jitter).toBe('decorrelated'); // not env-overridable
+  });
+
+  test('GBRAIN_BULK_MAX_RETRIES=0: accepted (debug-mode disable)', () => {
+    const out = resolveBulkRetryOpts({ GBRAIN_BULK_MAX_RETRIES: '0' });
+    expect(out.maxRetries).toBe(0);
+  });
+
+  test('GBRAIN_BULK_MAX_RETRIES=negative: throws with paste-ready hint', () => {
+    expect(() => resolveBulkRetryOpts({ GBRAIN_BULK_MAX_RETRIES: '-1' }))
+      .toThrow(/GBRAIN_BULK_MAX_RETRIES.*>= 0.*Fix: export/);
+  });
+
+  test('GBRAIN_BULK_MAX_RETRIES=non-int: throws', () => {
+    expect(() => resolveBulkRetryOpts({ GBRAIN_BULK_MAX_RETRIES: '3.5' }))
+      .toThrow(/GBRAIN_BULK_MAX_RETRIES/);
+    expect(() => resolveBulkRetryOpts({ GBRAIN_BULK_MAX_RETRIES: 'foo' }))
+      .toThrow(/GBRAIN_BULK_MAX_RETRIES/);
+  });
+
+  test('GBRAIN_BULK_RETRY_BASE_MS=0 or negative: throws (delays must be > 0)', () => {
+    expect(() => resolveBulkRetryOpts({ GBRAIN_BULK_RETRY_BASE_MS: '0' }))
+      .toThrow(/> 0/);
+    expect(() => resolveBulkRetryOpts({ GBRAIN_BULK_RETRY_BASE_MS: '-100' }))
+      .toThrow(/> 0/);
+  });
+
+  test('GBRAIN_BULK_RETRY_MAX_MS < base: throws', () => {
+    expect(() => resolveBulkRetryOpts({
+      GBRAIN_BULK_RETRY_BASE_MS: '5000',
+      GBRAIN_BULK_RETRY_MAX_MS: '3000',
+    })).toThrow(/>= GBRAIN_BULK_RETRY_BASE_MS=5000/);
+  });
+
+  test('empty string env values treated as unset', () => {
+    const out = resolveBulkRetryOpts({ GBRAIN_BULK_MAX_RETRIES: '' });
+    expect(out.maxRetries).toBe(BULK_RETRY_OPTS.maxRetries);
+  });
+});
+
+describe('BULK_RETRY_OPTS — Supavisor-tuned defaults shape', () => {
+  test('total worst-case wait covers 5-10s circuit-breaker window', () => {
+    // Compute the maximum possible cumulative wait with 'none' jitter as
+    // the upper bound (decorrelated jitter has tighter bound on average).
+    // base=1000, base*2=2000, base*4=4000, base*8=8000 (capped at maxDelay=10000)
+    // Sum across 3 retries: 1000+2000+4000=7000. With one more cap iteration
+    // we'd hit 8000 but we stop at maxRetries=3.
+    expect(BULK_RETRY_OPTS.maxRetries).toBe(3);
+    expect(BULK_RETRY_OPTS.delayMs).toBe(1000);
+    expect(BULK_RETRY_OPTS.delayMaxMs).toBe(10_000);
+    expect(BULK_RETRY_OPTS.jitter).toBe('decorrelated');
+  });
+
+  test('decorrelated jitter realized worst-case >= 8s across 3 retries', () => {
+    // Run 1000 trials, check that the realized cumulative wait reaches
+    // at least 8s in worst-case (a Supavisor 5-10s recovery window).
+    let maxObserved = 0;
+    for (let trial = 0; trial < 1000; trial++) {
+      let prev = 0;
+      let total = 0;
+      for (let attempt = 0; attempt < 3; attempt++) {
+        const d = computeNextDelay(attempt, prev, 1000, 10_000, 'decorrelated');
+        prev = d;
+        total += d;
+      }
+      if (total > maxObserved) maxObserved = total;
+    }
+    expect(maxObserved).toBeGreaterThanOrEqual(8000);
+  });
+});

test/doctor-batch-retry.test.ts

@@ -0,0 +1,156 @@
+// v0.41.18.0 — batch_retry_health doctor check (codex H-9 thresholds).
+//
+// Hermetic: never touches a real engine. Stubs the audit-writer read by
+// pointing GBRAIN_AUDIT_DIR at a tempdir and writing synthetic events.
+
+import { describe, expect, test, beforeEach, afterEach } from 'bun:test';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { withEnv } from './helpers/with-env.ts';
+import { checkBatchRetryHealth } from '../src/commands/doctor.ts';
+import type { BrainEngine } from '../src/core/engine.ts';
+import { BATCH_RETRY_FEATURE_NAME } from '../src/core/audit/batch-retry-audit.ts';
+
+// Minimal stub engine — checkBatchRetryHealth doesn't use it (the audit
+// read is via filesystem). Cast suppresses BrainEngine's many required
+// methods we don't need here.
+const stubEngine = {} as BrainEngine;
+
+let tmpDir: string;
+
+beforeEach(() => {
+  tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'doctor-batch-retry-'));
+});
+
+afterEach(() => {
+  try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* */ }
+});
+
+function writeEvent(now: Date, event: Record<string, unknown>) {
+  const filename = `${BATCH_RETRY_FEATURE_NAME}-${now.getUTCFullYear()}-W${String(getIsoWeek(now)).padStart(2, '0')}.jsonl`;
+  const filePath = path.join(tmpDir, filename);
+  fs.appendFileSync(filePath, JSON.stringify({ ts: now.toISOString(), ...event }) + '\n');
+}
+
+describe('checkBatchRetryHealth — ok states', () => {
+  test('no events in window = ok', async () => {
+    await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
+      const check = await checkBatchRetryHealth(stubEngine);
+      expect(check.status).toBe('ok');
+      expect(check.message).toContain('No exhausted batch retries');
+    });
+  });
+
+  test('only successful retries = ok with recovery count', async () => {
+    await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
+      const now = new Date();
+      writeEvent(now, { site: 'extract.links_inc', batch_size: 100, attempt: 1, outcome: 'success', delay_ms: 1000, error_message_summary: 'blip' });
+      writeEvent(now, { site: 'extract.links_inc', batch_size: 100, attempt: 2, outcome: 'success', delay_ms: 3000, error_message_summary: 'blip' });
+      const check = await checkBatchRetryHealth(stubEngine);
+      expect(check.status).toBe('ok');
+      expect(check.message).toContain('transient retry');
+    });
+  });
+
+  test('1-2 exhausted from same site (under per-site threshold of 3) = ok', async () => {
+    await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
+      const now = new Date();
+      writeEvent(now, { site: 'extract.links_inc', batch_size: 100, attempt: 4, outcome: 'exhausted', delay_ms: 0, error_message_summary: 'breaker' });
+      writeEvent(now, { site: 'extract.links_inc', batch_size: 100, attempt: 4, outcome: 'exhausted', delay_ms: 0, error_message_summary: 'breaker' });
+      const check = await checkBatchRetryHealth(stubEngine);
+      expect(check.status).toBe('ok');
+      expect(check.message).toContain('below per-site threshold');
+    });
+  });
+});
+
+describe('checkBatchRetryHealth — warn states (codex H-9 thresholds)', () => {
+  test('>=3 exhausted from same site in 24h = warn', async () => {
+    await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
+      const now = new Date();
+      for (let i = 0; i < 3; i++) {
+        writeEvent(now, { site: 'extract.links_inc', batch_size: 100, attempt: 4, outcome: 'exhausted', delay_ms: 0, error_message_summary: 'breaker' });
+      }
+      const check = await checkBatchRetryHealth(stubEngine);
+      expect(check.status).toBe('warn');
+      expect(check.message).toContain('extract.links_inc');
+      expect(check.message).toContain('GBRAIN_BULK_MAX_RETRIES');
+    });
+  });
+
+  test('>=5 cross-site exhausted in 24h = warn', async () => {
+    await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
+      const now = new Date();
+      // 2 from one site, 2 from another, 1 from a third = 5 total, none >=3 per site.
+      writeEvent(now, { site: 'extract.links_inc', batch_size: 100, attempt: 4, outcome: 'exhausted', delay_ms: 0, error_message_summary: 'a' });
+      writeEvent(now, { site: 'extract.links_inc', batch_size: 100, attempt: 4, outcome: 'exhausted', delay_ms: 0, error_message_summary: 'a' });
+      writeEvent(now, { site: 'extract.timeline_fs', batch_size: 50, attempt: 4, outcome: 'exhausted', delay_ms: 0, error_message_summary: 'a' });
+      writeEvent(now, { site: 'extract.timeline_fs', batch_size: 50, attempt: 4, outcome: 'exhausted', delay_ms: 0, error_message_summary: 'a' });
+      writeEvent(now, { site: 'mcp.put_page.autolink', batch_size: 25, attempt: 4, outcome: 'exhausted', delay_ms: 0, error_message_summary: 'a' });
+      const check = await checkBatchRetryHealth(stubEngine);
+      expect(check.status).toBe('warn');
+    });
+  });
+});
+
+describe('checkBatchRetryHealth — fail state', () => {
+  test('>=20 exhausted in 24h = fail (sustained breaker)', async () => {
+    await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
+      const now = new Date();
+      for (let i = 0; i < 20; i++) {
+        writeEvent(now, { site: 'extract.links_inc', batch_size: 100, attempt: 4, outcome: 'exhausted', delay_ms: 0, error_message_summary: 'breaker' });
+      }
+      const check = await checkBatchRetryHealth(stubEngine);
+      expect(check.status).toBe('fail');
+      expect(check.message).toContain('Sustained circuit-breaker');
+    });
+  });
+});
+
+describe('checkBatchRetryHealth — codex M-10 env validation at doctor time', () => {
+  test('invalid GBRAIN_BULK_MAX_RETRIES surfaces at doctor time with paste-ready hint', async () => {
+    await withEnv({ GBRAIN_AUDIT_DIR: tmpDir, GBRAIN_BULK_MAX_RETRIES: '-1' }, async () => {
+      const check = await checkBatchRetryHealth(stubEngine);
+      expect(check.status).toBe('warn');
+      expect(check.message).toContain('GBRAIN_BULK_*');
+      expect(check.message).toContain('export GBRAIN_BULK_MAX_RETRIES');
+    });
+  });
+
+  test('valid GBRAIN_BULK_MAX_RETRIES=0 (debug-mode disable) is accepted', async () => {
+    await withEnv({ GBRAIN_AUDIT_DIR: tmpDir, GBRAIN_BULK_MAX_RETRIES: '0' }, async () => {
+      const check = await checkBatchRetryHealth(stubEngine);
+      expect(check.status).toBe('ok'); // 0 retries is valid; no exhausted events either
+    });
+  });
+});
+
+describe('checkBatchRetryHealth — codex H-9 corruption tolerance', () => {
+  test('corrupted JSONL lines are counted, not crashed-on', async () => {
+    await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => {
+      const now = new Date();
+      writeEvent(now, { site: 'extract.links_inc', batch_size: 100, attempt: 1, outcome: 'success', delay_ms: 1000, error_message_summary: 'a' });
+      const filename = `${BATCH_RETRY_FEATURE_NAME}-${now.getUTCFullYear()}-W${String(getIsoWeek(now)).padStart(2, '0')}.jsonl`;
+      const filePath = path.join(tmpDir, filename);
+      fs.appendFileSync(filePath, '{not json}\nstill not\n');
+      const check = await checkBatchRetryHealth(stubEngine);
+      // Successful retry only, no exhausted events = ok. The corrupt count
+      // appears in the message as a note.
+      expect(check.status).toBe('ok');
+      expect(check.message).toContain('corrupt JSONL');
+    });
+  });
+});
+
+function getIsoWeek(d: Date): number {
+  const target = new Date(d.valueOf());
+  const dayNumber = (d.getUTCDay() + 6) % 7;
+  target.setUTCDate(target.getUTCDate() - dayNumber + 3);
+  const firstThursday = target.valueOf();
+  target.setUTCMonth(0, 1);
+  if (target.getUTCDay() !== 4) {
+    target.setUTCMonth(0, 1 + ((4 - target.getUTCDay()) + 7) % 7);
+  }
+  return 1 + Math.ceil((firstThursday - target.valueOf()) / 604800000);
+}

test/extract-batch-retry.test.ts

@@ -1,276 +0,0 @@
-// v0.41.2.1 — withRetry + logBatchRetry + snapshot-before-clear contract.
-//
-// Pinned contracts:
-//   - withRetry is a pure primitive (no UI), exports cleanly so tests
-//     import it without going through the 6 private flush() closures.
-//   - Classification uses isRetryableConnError from src/core/retry-matcher.ts
-//     (single source of truth; no inline pattern duplication).
-//   - Retry-matcher extension recognizes gbrain's GBrainError shape
-//     ({problem: 'No database connection'}) AND the literal message
-//     pattern. PR #1416's reported batch-loss incident is closed.
-//   - logBatchRetry writes stderr only when jsonMode is false.
-//   - Snapshot contract: batch.slice() BEFORE batch.length=0 so producer
-//     mutation during the 500ms retry delay can't lose items on retry.
-//
-// Hermetic: no engine, no PGLite, no env mutation, no DATABASE_URL.
-
-import { describe, expect, test, beforeEach } from 'bun:test';
-import { withRetry, logBatchRetry } from '../src/commands/extract.ts';
-import { isRetryableConnError } from '../src/core/retry-matcher.ts';
-
-// Minimal GBrainError shape — mirror the typed problem/detail fields used
-// by db.ts:getConnection so the retry-matcher extension recognizes it.
-class FakeGBrainError extends Error {
-  problem: string;
-  detail: string;
-  constructor(problem: string, detail: string) {
-    super(`${problem}: ${detail}`);
-    this.problem = problem;
-    this.detail = detail;
-  }
-}
-
-describe('isRetryableConnError extension (v0.41.2.1)', () => {
-  test('GBrainError with problem="No database connection" is retryable', () => {
-    const err = new FakeGBrainError('No database connection', 'connect() has not been called');
-    expect(isRetryableConnError(err)).toBe(true);
-  });
-
-  test('GBrainError with other problem is NOT retryable', () => {
-    const err = new FakeGBrainError('Schema mismatch', 'expected vector(1536), got vector(1024)');
-    expect(isRetryableConnError(err)).toBe(false);
-  });
-
-  test('plain Error with "No database connection" message is retryable (literal match)', () => {
-    expect(isRetryableConnError(new Error('No database connection: connect() has not been called.'))).toBe(true);
-  });
-
-  test('constraint violation 23505 is NOT retryable', () => {
-    const err = Object.assign(new Error('duplicate key value violates unique constraint'), { code: '23505' });
-    expect(isRetryableConnError(err)).toBe(false);
-  });
-});
-
-describe('withRetry primitive (v0.41.2.1)', () => {
-  test('first-call success: returns value, no onRetry invocation', async () => {
-    let calls = 0;
-    let retried = false;
-    const result = await withRetry(
-      async () => { calls++; return 42; },
-      { onRetry: () => { retried = true; }, delayMs: 0 },
-    );
-    expect(result).toBe(42);
-    expect(calls).toBe(1);
-    expect(retried).toBe(false);
-  });
-
-  test('retries on Connection terminated; second attempt succeeds', async () => {
-    let calls = 0;
-    const result = await withRetry(
-      async () => {
-        calls++;
-        if (calls === 1) throw new Error('Connection terminated unexpectedly');
-        return 'recovered';
-      },
-      { delayMs: 0 },
-    );
-    expect(result).toBe('recovered');
-    expect(calls).toBe(2);
-  });
-
-  test('retries on GBrainError "No database connection"; second succeeds', async () => {
-    let calls = 0;
-    const result = await withRetry(
-      async () => {
-        calls++;
-        if (calls === 1) throw new FakeGBrainError('No database connection', 'connect() has not been called');
-        return 'ok';
-      },
-      { delayMs: 0 },
-    );
-    expect(result).toBe('ok');
-    expect(calls).toBe(2);
-  });
-
-  test('non-retryable error propagates immediately, no retry', async () => {
-    let calls = 0;
-    const promise = withRetry(
-      async () => {
-        calls++;
-        const err = Object.assign(new Error('duplicate key'), { code: '23505' });
-        throw err;
-      },
-      { delayMs: 0 },
-    );
-    await expect(promise).rejects.toThrow('duplicate key');
-    expect(calls).toBe(1); // no retry on 23505
-  });
-
-  test('second failure propagates (single retry, not infinite)', async () => {
-    let calls = 0;
-    const promise = withRetry(
-      async () => {
-        calls++;
-        throw new Error('ECONNRESET');
-      },
-      { delayMs: 0 },
-    );
-    await expect(promise).rejects.toThrow('ECONNRESET');
-    expect(calls).toBe(2); // attempt 1 + retry, then propagate
-  });
-
-  test('onRetry callback receives (attempt=1, err)', async () => {
-    let received: { attempt: number; err: unknown } | null = null;
-    let calls = 0;
-    await withRetry(
-      async () => {
-        calls++;
-        if (calls === 1) throw new Error('Connection terminated unexpectedly');
-        return null;
-      },
-      {
-        onRetry: (attempt, err) => { received = { attempt, err }; },
-        delayMs: 0,
-      },
-    );
-    expect(received).not.toBeNull();
-    expect(received!.attempt).toBe(1);
-    expect(received!.err).toBeInstanceOf(Error);
-    expect((received!.err as Error).message).toBe('Connection terminated unexpectedly');
-  });
-
-  test('delayMs default is 500ms when not specified', async () => {
-    let calls = 0;
-    const start = Date.now();
-    await withRetry(
-      async () => {
-        calls++;
-        if (calls === 1) throw new Error('ECONNRESET');
-        return null;
-      },
-      // no delayMs override
-    );
-    const elapsed = Date.now() - start;
-    expect(calls).toBe(2);
-    // Allow ±50ms tolerance for scheduler jitter
-    expect(elapsed).toBeGreaterThanOrEqual(450);
-    expect(elapsed).toBeLessThan(2000); // never close to forever
-  }, 5000);
-});
-
-describe('logBatchRetry helper (v0.41.2.1)', () => {
-  let stderrCaptured: string[] = [];
-  let originalStderr: typeof console.error;
-
-  beforeEach(() => {
-    stderrCaptured = [];
-    originalStderr = console.error;
-    console.error = (...args: unknown[]) => {
-      stderrCaptured.push(args.map(String).join(' '));
-    };
-  });
-
-  test('writes single stderr line when jsonMode=false', () => {
-    logBatchRetry('extract.links_fs', 73, new Error('Connection terminated'), false);
-    try {
-      expect(stderrCaptured).toHaveLength(1);
-      expect(stderrCaptured[0]).toContain('extract.links_fs');
-      expect(stderrCaptured[0]).toContain('73');
-      expect(stderrCaptured[0]).toContain('Connection terminated');
-    } finally {
-      console.error = originalStderr;
-    }
-  });
-
-  test('writes NOTHING when jsonMode=true', () => {
-    logBatchRetry('extract.links_fs', 73, new Error('ECONNRESET'), true);
-    try {
-      expect(stderrCaptured).toHaveLength(0);
-    } finally {
-      console.error = originalStderr;
-    }
-  });
-
-  test('handles non-Error throwables by stringifying', () => {
-    logBatchRetry('extract.timeline_db', 12, 'string-error', false);
-    try {
-      expect(stderrCaptured).toHaveLength(1);
-      expect(stderrCaptured[0]).toContain('string-error');
-    } finally {
-      console.error = originalStderr;
-    }
-  });
-});
-
-describe('snapshot-before-clear contract (load-bearing for PR #1416 fix)', () => {
-  // This contract is what the 6 flush() sites depend on. The retry primitive
-  // doesn't enforce it itself — the call site must snapshot the batch BEFORE
-  // clearing it, so a producer pushing during the retry delay can't lose
-  // items on retry. We simulate the flush() pattern directly.
-
-  test('snapshot is what retry sends, NOT the post-clear batch', async () => {
-    let batch: number[] = [1, 2, 3, 4, 5];
-    let receivedOnAttempt: number[][] = [];
-
-    async function flushPattern() {
-      if (batch.length === 0) return;
-      // Snapshot-before-clear, as flush sites in extract.ts do
-      const snapshot = batch.slice();
-      batch.length = 0;
-      let calls = 0;
-      await withRetry(
-        async () => {
-          calls++;
-          receivedOnAttempt.push(snapshot.slice());
-          if (calls === 1) {
-            // Producer mutates `batch` during the retry delay (simulated by
-            // pushing items right now — the retry hasn't fired yet, but in
-            // real code the producer's next iteration writes here).
-            batch.push(99, 100, 101);
-            throw new Error('Connection terminated unexpectedly');
-          }
-          return snapshot.length;
-        },
-        { delayMs: 0 },
-      );
-    }
-
-    await flushPattern();
-
-    // Both attempts must have received the SAME snapshot — not the
-    // post-clear batch state. If snapshot-before-clear is broken,
-    // attempt 2 sees [99, 100, 101] (producer's new items only).
-    expect(receivedOnAttempt).toHaveLength(2);
-    expect(receivedOnAttempt[0]).toEqual([1, 2, 3, 4, 5]);
-    expect(receivedOnAttempt[1]).toEqual([1, 2, 3, 4, 5]); // retry sends snapshot, not batch
-    // batch contains the producer's new items, ready for next flush
-    expect(batch).toEqual([99, 100, 101]);
-  });
-
-  test('error message uses snapshot length, not post-clear batch length', async () => {
-    let batch: number[] = [10, 20, 30];
-    let capturedErrorMsg = '';
-
-    async function flushPattern() {
-      if (batch.length === 0) return;
-      const snapshot = batch.slice();
-      batch.length = 0; // batch.length is now 0
-      try {
-        await withRetry(
-          async () => {
-            throw new Error('ECONNRESET'); // both attempts fail
-          },
-          { delayMs: 0 },
-        );
-      } catch (e) {
-        const msg = e instanceof Error ? e.message : String(e);
-        // The CONTRACT: error message reads snapshot.length, NOT batch.length
-        capturedErrorMsg = `batch error (${snapshot.length} rows lost): ${msg}`;
-      }
-    }
-
-    await flushPattern();
-    expect(capturedErrorMsg).toContain('3 rows lost'); // snapshot had 3, not batch's 0
-    expect(capturedErrorMsg).toContain('ECONNRESET');
-  });
-});