mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-31 04:07:52 +00:00
* fix(minions): self-identifying RSS watchdog + cgroup-aware default + pooler-reap self-heal (#1678) Problem 1: distinct WORKER_EXIT_RSS_WATCHDOG exit code + cause-keyed supervisor breaker (bypasses the stable-run reset that hid the 400x/24h loop) + rss_watchdog audit bucket + 80% soft-warn; cgroup-aware resolveDefaultMaxRssMb replaces the flat 2048 default at every spawn site. Problem 2: CONNECTION_ENDED classified retryable; postgres-engine sql getter throws a retryable error on a reaped instance pool instead of the misleading module-singleton fallthrough; promoteDelayed reconnect-retry; claim recovers on the next poll tick (no double-claim); lock-renewal tick reconnect-once dep. * feat(cycle): surface silent extract_atoms backlog + bounded --drain + fix lint clobbering the shared DB connection (#1678) Problem 3: extract_atoms_backlog doctor check + pack_gated skip marker + shared countExtractAtomsBacklog; `gbrain dream --phase extract_atoms --drain [--window N]` single-hold bounded drain (same cycleLockIdFor, rediscover each batch, reports remaining, exits non-zero while work remains). Also fixes a real production bug found via E2E: the cycle lint phase's resolveLintContentSanity created + disconnected a module-style engine that nulled the shared db singleton mid-cycle, breaking every later phase with "connect() has not been called". Lint now reuses the caller's live engine (cycle + Minion handlers thread it; standalone CLI keeps the create-own path). * chore: bump version and changelog (v0.41.39.0) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(#1678): pre-landing review — route transaction/withReservedConnection through the sql getter + drain treats failed count as incomplete Codex adversarial review findings: - #2: transaction(), withReservedConnection(), and one other site bypassed the v0.42.2.0 sql-getter self-heal via `this._sql || db.getConnection()`, so a reaped instance pool fell through to the module singleton there. Route all three through `this.sql` so they throw the retryable instance-pool error and recover consistently (MinionQueue.transaction hits this). - #4: `gbrain dream --drain` treated a null backlog count (query failure) as success via `remaining ?? 0`; now null exits EXIT_DRAIN_INCOMPLETE so automation never believes an unverified backlog drained. - #1 (claim orphan) + #3 (PGLite drain lock) documented as follow-ups in TODOS. * docs: document v0.42.2.0 #1678 modules + behavior in CLAUDE.md Adds Key Files entries for worker-exit-codes.ts, rss-default.ts, and extract-atoms-drain.ts, plus v0.42.2.0 annotations on worker.ts, child-worker-supervisor.ts, lock-renewal-tick.ts, and dream.ts. Regenerated llms-full.txt to match (test/build-llms.test.ts gate). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * chore: re-version v0.42.2.0 → v0.42.5.0 across VERSION/package.json/CHANGELOG/docs/comments Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
109 lines
3.9 KiB
TypeScript
109 lines
3.9 KiB
TypeScript
import { describe, expect, test } from 'bun:test';
|
|
import {
|
|
isStatementTimeoutError,
|
|
isLockTimeoutError,
|
|
isRetryableConnError,
|
|
isRetryableError,
|
|
} from '../src/core/retry-matcher.ts';
|
|
|
|
function pgError(code: string, message: string): Error & { code: string } {
|
|
const err = new Error(message) as Error & { code: string };
|
|
err.code = code;
|
|
return err;
|
|
}
|
|
|
|
describe('isStatementTimeoutError', () => {
|
|
test('matches SQLSTATE 57014', () => {
|
|
expect(isStatementTimeoutError(pgError('57014', 'canceled'))).toBe(true);
|
|
});
|
|
|
|
test('matches the canceling-statement message', () => {
|
|
expect(
|
|
isStatementTimeoutError(new Error('canceling statement due to statement timeout'))
|
|
).toBe(true);
|
|
});
|
|
|
|
test('does not match other errors', () => {
|
|
expect(isStatementTimeoutError(new Error('connection refused'))).toBe(false);
|
|
expect(isStatementTimeoutError(pgError('08006', 'connection_failure'))).toBe(false);
|
|
});
|
|
});
|
|
|
|
describe('isLockTimeoutError', () => {
|
|
test('matches SQLSTATE 55P03', () => {
|
|
expect(isLockTimeoutError(pgError('55P03', 'lock not available'))).toBe(true);
|
|
});
|
|
|
|
test('matches lock_not_available message', () => {
|
|
expect(isLockTimeoutError(new Error('could not obtain lock on row'))).toBe(true);
|
|
});
|
|
|
|
test('does not match statement timeouts', () => {
|
|
expect(isLockTimeoutError(pgError('57014', 'canceled'))).toBe(false);
|
|
});
|
|
});
|
|
|
|
describe('isRetryableConnError', () => {
|
|
test('matches Postgres class 08 codes', () => {
|
|
expect(isRetryableConnError(pgError('08000', 'connection_exception'))).toBe(true);
|
|
expect(isRetryableConnError(pgError('08003', 'connection_does_not_exist'))).toBe(true);
|
|
expect(isRetryableConnError(pgError('08006', 'connection_failure'))).toBe(true);
|
|
});
|
|
|
|
test('matches connection-refused message', () => {
|
|
expect(isRetryableConnError(new Error('connection refused'))).toBe(true);
|
|
});
|
|
|
|
test('matches ECONNRESET', () => {
|
|
expect(isRetryableConnError(new Error('ECONNRESET'))).toBe(true);
|
|
});
|
|
|
|
test('matches database-starting-up', () => {
|
|
expect(
|
|
isRetryableConnError(new Error('the database system is starting up'))
|
|
).toBe(true);
|
|
});
|
|
|
|
test('does NOT match statement timeouts', () => {
|
|
expect(isRetryableConnError(pgError('57014', 'canceled'))).toBe(false);
|
|
});
|
|
|
|
test('does NOT match lock timeouts', () => {
|
|
expect(isRetryableConnError(pgError('55P03', 'lock'))).toBe(false);
|
|
});
|
|
|
|
test('does not match arbitrary errors', () => {
|
|
expect(isRetryableConnError(new Error('something else'))).toBe(false);
|
|
});
|
|
|
|
// issue #1678: postgres.js's transaction-mode pooler reaps idle sockets and
|
|
// throws errors carrying `code: 'CONNECTION_ENDED'` (a library code, not an
|
|
// 08xxx SQLSTATE). Must be retryable via BOTH the code and the message form.
|
|
test('matches CONNECTION_ENDED via code', () => {
|
|
expect(isRetryableConnError(pgError('CONNECTION_ENDED', 'write CONNECTION_ENDED'))).toBe(true);
|
|
});
|
|
|
|
test('matches CONNECTION_ENDED via message even without the code', () => {
|
|
expect(isRetryableConnError(new Error('write CONNECTION_ENDED localhost:6543'))).toBe(true);
|
|
});
|
|
|
|
// The getter self-heal throws a GBrainError whose `problem` field is
|
|
// 'No database connection' — the existing typed-shape match must keep firing.
|
|
test('matches the instance-pool-reaped GBrainError shape (problem field)', () => {
|
|
const err = { problem: 'No database connection', message: 'instance pool torn down' };
|
|
expect(isRetryableConnError(err)).toBe(true);
|
|
});
|
|
});
|
|
|
|
describe('isRetryableError', () => {
|
|
test('union: returns true for conn AND statement-timeout', () => {
|
|
expect(isRetryableError(new Error('connection refused'))).toBe(true);
|
|
expect(isRetryableError(pgError('57014', 'canceled'))).toBe(true);
|
|
expect(isRetryableError(new Error('ECONNRESET'))).toBe(true);
|
|
});
|
|
|
|
test('still false for unrelated errors', () => {
|
|
expect(isRetryableError(new Error('foreign key violation'))).toBe(false);
|
|
});
|
|
});
|