gbrain/test/core/retry.test.ts

// v0.41.18.0 — src/core/retry.ts canonical retry primitive.
//
// Moved from test/extract-batch-retry.test.ts when withRetry was factored
// out of extract.ts (Eng-D2 architectural pivot: engine-level wrap instead
// of per-call-site wrap). All v0.41.2.1 contracts preserved verbatim:
//
//   - withRetry is a pure primitive (no UI), exports cleanly.
//   - Classification uses isRetryableConnError from retry-matcher.ts.
//   - Default maxRetries=1 = single 500ms retry (v0.41.2.1 back-compat).
//   - Non-retryable errors propagate immediately.
//   - onRetry callback fires per attempt with (1-based attempt, err).
//
// v0.41.18.0 codex-hardening additions:
//   - BULK_RETRY_OPTS defaults (3 retries, 1s/3s/8s exponential w/ decorrelated jitter)
//   - AbortSignal threading + abortableSleep
//   - Typed BatchAuditSite enum + isBatchAuditSite guard
//   - resolveBulkRetryOpts env-override with paste-ready error hints
//   - computeNextDelay pure-function for each jitter mode
//
// Hermetic: no engine, no PGLite, no env mutation (uses withEnv helper),
// no DATABASE_URL required.

import { describe, expect, test, beforeEach } from 'bun:test';
import {
  withRetry,
  abortableSleep,
  computeNextDelay,
  resolveBulkRetryOpts,
  isBatchAuditSite,
  isRetryableConnError,
  BULK_RETRY_OPTS,
  BATCH_AUDIT_SITES,
  RetryAbortError,
} from '../../src/core/retry.ts';

// Minimal GBrainError shape mirrors the typed problem/detail fields from
// db.ts:getConnection so the retry-matcher extension recognizes it.
class FakeGBrainError extends Error {
  problem: string;
  detail: string;
  constructor(problem: string, detail: string) {
    super(`${problem}: ${detail}`);
    this.problem = problem;
    this.detail = detail;
  }
}

describe('isRetryableConnError extension (v0.41.2.1, re-exported from retry.ts)', () => {
  test('GBrainError with problem="No database connection" is retryable', () => {
    const err = new FakeGBrainError('No database connection', 'connect() has not been called');
    expect(isRetryableConnError(err)).toBe(true);
  });

  test('GBrainError with other problem is NOT retryable', () => {
    const err = new FakeGBrainError('Schema mismatch', 'expected vector(1536), got vector(1024)');
    expect(isRetryableConnError(err)).toBe(false);
  });

  test('plain Error with "No database connection" message is retryable (literal match)', () => {
    expect(isRetryableConnError(new Error('No database connection: connect() has not been called.'))).toBe(true);
  });

  test('constraint violation 23505 is NOT retryable', () => {
    const err = Object.assign(new Error('duplicate key value violates unique constraint'), { code: '23505' });
    expect(isRetryableConnError(err)).toBe(false);
  });
});

describe('withRetry primitive — v0.41.2.1 back-compat contract', () => {
  test('first-call success: returns value, no onRetry invocation', async () => {
    let calls = 0;
    let retried = false;
    const result = await withRetry(
      async () => { calls++; return 42; },
      { onRetry: () => { retried = true; }, delayMs: 0 },
    );
    expect(result).toBe(42);
    expect(calls).toBe(1);
    expect(retried).toBe(false);
  });

  test('retries on Connection terminated; second attempt succeeds', async () => {
    let calls = 0;
    const result = await withRetry(
      async () => {
        calls++;
        if (calls === 1) throw new Error('Connection terminated unexpectedly');
        return 'recovered';
      },
      { delayMs: 0 },
    );
    expect(result).toBe('recovered');
    expect(calls).toBe(2);
  });

  test('retries on GBrainError "No database connection"; second succeeds', async () => {
    let calls = 0;
    const result = await withRetry(
      async () => {
        calls++;
        if (calls === 1) throw new FakeGBrainError('No database connection', 'connect() has not been called');
        return 'ok';
      },
      { delayMs: 0 },
    );
    expect(result).toBe('ok');
    expect(calls).toBe(2);
  });

  test('non-retryable error propagates immediately, no retry', async () => {
    let calls = 0;
    const promise = withRetry(
      async () => {
        calls++;
        const err = Object.assign(new Error('duplicate key'), { code: '23505' });
        throw err;
      },
      { delayMs: 0 },
    );
    await expect(promise).rejects.toThrow('duplicate key');
    expect(calls).toBe(1); // no retry on 23505
  });

  test('default maxRetries=1: second failure propagates (single retry, not infinite)', async () => {
    let calls = 0;
    const promise = withRetry(
      async () => {
        calls++;
        throw new Error('ECONNRESET');
      },
      { delayMs: 0 },
    );
    await expect(promise).rejects.toThrow('ECONNRESET');
    expect(calls).toBe(2); // attempt 1 + 1 retry, then propagate (back-compat)
  });

  test('onRetry callback receives (attempt=1, err)', async () => {
    let received: { attempt: number; err: unknown } | null = null;
    let calls = 0;
    await withRetry(
      async () => {
        calls++;
        if (calls === 1) throw new Error('Connection terminated unexpectedly');
        return null;
      },
      {
        onRetry: (attempt, err) => { received = { attempt, err }; },
        delayMs: 0,
      },
    );
    expect(received).not.toBeNull();
    expect(received!.attempt).toBe(1);
    expect(received!.err).toBeInstanceOf(Error);
    expect((received!.err as Error).message).toBe('Connection terminated unexpectedly');
  });

  test('delayMs default is 500ms when not specified', async () => {
    let calls = 0;
    const start = Date.now();
    await withRetry(
      async () => {
        calls++;
        if (calls === 1) throw new Error('ECONNRESET');
        return null;
      },
      // no delayMs override
    );
    const elapsed = Date.now() - start;
    expect(calls).toBe(2);
    expect(elapsed).toBeGreaterThanOrEqual(450); // ±50ms scheduler tolerance
    expect(elapsed).toBeLessThan(2000);
  }, 5000);
});

describe('withRetry — maxRetries > 1 (v0.41.18.0 BULK_RETRY_OPTS contract)', () => {
  test('maxRetries=3: succeeds on third retry', async () => {
    let calls = 0;
    const result = await withRetry(
      async () => {
        calls++;
        if (calls <= 3) throw new Error('Connection terminated unexpectedly');
        return 'recovered';
      },
      { delayMs: 0, maxRetries: 3 },
    );
    expect(result).toBe('recovered');
    expect(calls).toBe(4); // 1 initial + 3 retries
  });

  test('maxRetries=3: all retries fail, propagates last error', async () => {
    let calls = 0;
    const promise = withRetry(
      async () => {
        calls++;
        throw new Error('ECONNRESET');
      },
      { delayMs: 0, maxRetries: 3 },
    );
    await expect(promise).rejects.toThrow('ECONNRESET');
    expect(calls).toBe(4); // 1 initial + 3 retries
  });

  test('maxRetries=0: no retry, single attempt only (operator-debug mode)', async () => {
    let calls = 0;
    const promise = withRetry(
      async () => {
        calls++;
        throw new Error('ECONNRESET');
      },
      { delayMs: 0, maxRetries: 0 },
    );
    await expect(promise).rejects.toThrow('ECONNRESET');
    expect(calls).toBe(1); // exhausted before retry fires
  });
});

describe('computeNextDelay — exponential / full / decorrelated jitter', () => {
  test('jitter=none: pure exponential capped at maxDelay', () => {
    expect(computeNextDelay(0, 0, 1000, 10_000, 'none')).toBe(1000);
    expect(computeNextDelay(1, 1000, 1000, 10_000, 'none')).toBe(2000);
    expect(computeNextDelay(2, 2000, 1000, 10_000, 'none')).toBe(4000);
    expect(computeNextDelay(3, 4000, 1000, 10_000, 'none')).toBe(8000);
    // cap kicks in
    expect(computeNextDelay(4, 8000, 1000, 10_000, 'none')).toBe(10_000);
  });

  test('jitter=full: uniform in [0, exponential]', () => {
    const rng = () => 0.5; // deterministic 50%
    expect(computeNextDelay(0, 0, 1000, 10_000, 'full', rng)).toBe(500);
    expect(computeNextDelay(1, 0, 1000, 10_000, 'full', rng)).toBe(1000);
    expect(computeNextDelay(2, 0, 1000, 10_000, 'full', rng)).toBe(2000);
  });

  test('jitter=decorrelated: uniform in [base, prevDelay*3] capped at maxDelay', () => {
    const rng = () => 0; // always low end → base
    expect(computeNextDelay(0, 0, 1000, 10_000, 'decorrelated', rng)).toBe(1000);
    expect(computeNextDelay(1, 1000, 1000, 10_000, 'decorrelated', rng)).toBe(1000);
    expect(computeNextDelay(2, 3000, 1000, 10_000, 'decorrelated', rng)).toBe(1000);

    const rngHigh = () => 0.99; // always high end → near cap
    const d = computeNextDelay(3, 3000, 1000, 10_000, 'decorrelated', rngHigh);
    // hi = max(base=1000, prevDelay*3=9000) = 9000
    expect(d).toBeGreaterThanOrEqual(8500);
    expect(d).toBeLessThanOrEqual(9000);
  });

  test('jitter=decorrelated: first retry (prevDelay=0) does NOT degenerate to base-only', () => {
    // The codex-caught initialization bug: when prevDelay=0, the formula
    // could pick uniform(base, max(base, 0)) = base every time. The fix
    // floors `hi` at `base` so we get uniform(base, base) = base on first
    // retry, which is at least the base delay — not zero.
    expect(computeNextDelay(0, 0, 1000, 10_000, 'decorrelated', () => 0)).toBe(1000);
    expect(computeNextDelay(0, 0, 1000, 10_000, 'decorrelated', () => 1)).toBe(1000);
  });

  test('jitter=decorrelated: NEVER produces near-zero (the codex C-2 fix)', () => {
    // The whole point of replacing 'full' with 'decorrelated': bad pooler
    // recovery happens when retries near-zero re-hit the breaker. Floor
    // at base prevents that.
    for (let i = 0; i < 100; i++) {
      const d = computeNextDelay(2, 3000, 1000, 10_000, 'decorrelated');
      expect(d).toBeGreaterThanOrEqual(1000);
    }
  });
});

describe('abortableSleep + AbortSignal threading (D9 codex)', () => {
  test('abortableSleep resolves after ms when no signal', async () => {
    const start = Date.now();
    await abortableSleep(50);
    const elapsed = Date.now() - start;
    expect(elapsed).toBeGreaterThanOrEqual(40);
    expect(elapsed).toBeLessThan(200);
  });

  test('abortableSleep rejects immediately when signal already aborted', async () => {
    const ctrl = new AbortController();
    ctrl.abort();
    await expect(abortableSleep(1000, ctrl.signal)).rejects.toBeInstanceOf(RetryAbortError);
  });

  test('abortableSleep rejects mid-sleep when signal fires', async () => {
    const ctrl = new AbortController();
    const start = Date.now();
    const sleepPromise = abortableSleep(5000, ctrl.signal);
    setTimeout(() => ctrl.abort(), 20);
    await expect(sleepPromise).rejects.toBeInstanceOf(RetryAbortError);
    const elapsed = Date.now() - start;
    expect(elapsed).toBeLessThan(500); // didn't wait the full 5s
  });

  test('withRetry honors signal: aborts mid-retry-sleep', async () => {
    const ctrl = new AbortController();
    let calls = 0;
    const promise = withRetry(
      async () => {
        calls++;
        throw new Error('ECONNRESET');
      },
      { maxRetries: 5, delayMs: 1000, signal: ctrl.signal },
    );
    setTimeout(() => ctrl.abort(), 50);
    await expect(promise).rejects.toBeInstanceOf(RetryAbortError);
    // Should abort before all 6 attempts completed (each would wait 1s+)
    expect(calls).toBeLessThan(6);
  });

  test('withRetry honors signal: aborts BEFORE first attempt if already aborted', async () => {
    const ctrl = new AbortController();
    ctrl.abort();
    let calls = 0;
    const promise = withRetry(
      async () => {
        calls++;
        return 'ok';
      },
      { signal: ctrl.signal },
    );
    await expect(promise).rejects.toBeInstanceOf(RetryAbortError);
    expect(calls).toBe(0); // never ran the function
  });
});

describe('BATCH_AUDIT_SITES typed enum + isBatchAuditSite guard (D10c codex)', () => {
  test('every known site is recognized', () => {
    for (const site of BATCH_AUDIT_SITES) {
      expect(isBatchAuditSite(site)).toBe(true);
    }
  });

  test('unknown sites rejected', () => {
    expect(isBatchAuditSite('extract.typo')).toBe(false);
    expect(isBatchAuditSite('mcp.put_pAge.autolink')).toBe(false);
    expect(isBatchAuditSite('')).toBe(false);
    expect(isBatchAuditSite('addLinksBatchz')).toBe(false);
  });

  test('list contains all CEO + eng + codex callers (regression: future deletes must be intentional)', () => {
    // Pin the set so a future "cleanup" PR can't silently drop a site and
    // break audit-attribution for the corresponding caller.
    const expected = new Set([
      'addLinksBatch', 'addTimelineEntriesBatch', 'addTakesBatch', 'upsertChunks',
      'extract.links_inc', 'extract.timeline_inc',
      'extract.links_fs', 'extract.timeline_fs',
      'extract.links_db', 'extract.timeline_db',
      'extract.by_mention',
      'extract.stale',
      'mcp.put_page.autolink',
      'sync.import_file',
      'reindex.markdown', 'reindex.multimodal',
      'backfill.outer',
      'minion-lock',
    ]);
    expect(new Set<string>([...BATCH_AUDIT_SITES])).toEqual(expected);
  });
});

describe('resolveBulkRetryOpts env-override (D3 cherry-pick, codex M-10/M-12)', () => {
  test('no env vars: returns BULK_RETRY_OPTS verbatim', () => {
    const out = resolveBulkRetryOpts({});
    expect(out).toEqual(BULK_RETRY_OPTS);
  });

  test('all 3 vars set: overrides apply', () => {
    const out = resolveBulkRetryOpts({
      GBRAIN_BULK_MAX_RETRIES: '5',
      GBRAIN_BULK_RETRY_BASE_MS: '2000',
      GBRAIN_BULK_RETRY_MAX_MS: '15000',
    });
    expect(out.maxRetries).toBe(5);
    expect(out.delayMs).toBe(2000);
    expect(out.delayMaxMs).toBe(15_000);
    expect(out.jitter).toBe('decorrelated'); // not env-overridable
  });

  test('GBRAIN_BULK_MAX_RETRIES=0: accepted (debug-mode disable)', () => {
    const out = resolveBulkRetryOpts({ GBRAIN_BULK_MAX_RETRIES: '0' });
    expect(out.maxRetries).toBe(0);
  });

  test('GBRAIN_BULK_MAX_RETRIES=negative: throws with paste-ready hint', () => {
    expect(() => resolveBulkRetryOpts({ GBRAIN_BULK_MAX_RETRIES: '-1' }))
      .toThrow(/GBRAIN_BULK_MAX_RETRIES.*>= 0.*Fix: export/);
  });

  test('GBRAIN_BULK_MAX_RETRIES=non-int: throws', () => {
    expect(() => resolveBulkRetryOpts({ GBRAIN_BULK_MAX_RETRIES: '3.5' }))
      .toThrow(/GBRAIN_BULK_MAX_RETRIES/);
    expect(() => resolveBulkRetryOpts({ GBRAIN_BULK_MAX_RETRIES: 'foo' }))
      .toThrow(/GBRAIN_BULK_MAX_RETRIES/);
  });

  test('GBRAIN_BULK_RETRY_BASE_MS=0 or negative: throws (delays must be > 0)', () => {
    expect(() => resolveBulkRetryOpts({ GBRAIN_BULK_RETRY_BASE_MS: '0' }))
      .toThrow(/> 0/);
    expect(() => resolveBulkRetryOpts({ GBRAIN_BULK_RETRY_BASE_MS: '-100' }))
      .toThrow(/> 0/);
  });

  test('GBRAIN_BULK_RETRY_MAX_MS < base: throws', () => {
    expect(() => resolveBulkRetryOpts({
      GBRAIN_BULK_RETRY_BASE_MS: '5000',
      GBRAIN_BULK_RETRY_MAX_MS: '3000',
    })).toThrow(/>= GBRAIN_BULK_RETRY_BASE_MS=5000/);
  });

  test('empty string env values treated as unset', () => {
    const out = resolveBulkRetryOpts({ GBRAIN_BULK_MAX_RETRIES: '' });
    expect(out.maxRetries).toBe(BULK_RETRY_OPTS.maxRetries);
  });
});

describe('BULK_RETRY_OPTS — Supavisor-tuned defaults shape', () => {
  test('total worst-case wait covers 5-10s circuit-breaker window', () => {
    // Compute the maximum possible cumulative wait with 'none' jitter as
    // the upper bound (decorrelated jitter has tighter bound on average).
    // base=1000, base*2=2000, base*4=4000, base*8=8000 (capped at maxDelay=10000)
    // Sum across 3 retries: 1000+2000+4000=7000. With one more cap iteration
    // we'd hit 8000 but we stop at maxRetries=3.
    expect(BULK_RETRY_OPTS.maxRetries).toBe(3);
    expect(BULK_RETRY_OPTS.delayMs).toBe(1000);
    expect(BULK_RETRY_OPTS.delayMaxMs).toBe(10_000);
    expect(BULK_RETRY_OPTS.jitter).toBe('decorrelated');
  });

  test('decorrelated jitter realized worst-case >= 8s across 3 retries', () => {
    // Run 1000 trials, check that the realized cumulative wait reaches
    // at least 8s in worst-case (a Supavisor 5-10s recovery window).
    let maxObserved = 0;
    for (let trial = 0; trial < 1000; trial++) {
      let prev = 0;
      let total = 0;
      for (let attempt = 0; attempt < 3; attempt++) {
        const d = computeNextDelay(attempt, prev, 1000, 10_000, 'decorrelated');
        prev = d;
        total += d;
      }
      if (total > maxObserved) maxObserved = total;
    }
    expect(maxObserved).toBeGreaterThanOrEqual(8000);
  });
});