mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
* fix(retry): match EMAXCONNSESSION + SQLSTATE 53300 as retryable conn errors (#1794) * feat(schema): add op_checkpoint_paths append-only delta table (migration v115) (#1794) * refactor(op-checkpoint): append-only deltas via executeRawDirect + withRetry (#1794) * fix(sync): resumable-checkpoint durability + lock-thrash fix (#1794) Durable append-only checkpoint writes (executeRawDirect + retry), fail-loud consecutive-failure abort, first-file/10s flush cadence, race-safe pending-delta under parallel workers, guaranteed final flush on every exit path incl. SIGTERM (no-retry one-shot via registerCleanup), bankedFiles/reason observability, event-loop yield to keep the lock heartbeat alive, and routing the bare (no-source) sync through withRefreshingLock. * fix(db-lock): heartbeat-aware takeover + direct-pool refresh (#1794) * fix(cycle): treat SyncLockBusyError as skip, not a phase failure (#1794) * docs(sync): document the 5 checkpoint/lock env knobs (#1794) * v0.42.36.0 fix(sync): resumable, durable, single-flight sync — converges under pool exhaustion + repeated kills (#1794) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * docs(key-files): update sync.ts + op-checkpoint.ts entries to resumable-checkpoint current state (#1794) --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
137 lines
5.2 KiB
TypeScript
137 lines
5.2 KiB
TypeScript
import { describe, expect, test } from 'bun:test';
|
|
import {
|
|
isStatementTimeoutError,
|
|
isLockTimeoutError,
|
|
isRetryableConnError,
|
|
isRetryableError,
|
|
} from '../src/core/retry-matcher.ts';
|
|
|
|
function pgError(code: string, message: string): Error & { code: string } {
|
|
const err = new Error(message) as Error & { code: string };
|
|
err.code = code;
|
|
return err;
|
|
}
|
|
|
|
describe('isStatementTimeoutError', () => {
|
|
test('matches SQLSTATE 57014', () => {
|
|
expect(isStatementTimeoutError(pgError('57014', 'canceled'))).toBe(true);
|
|
});
|
|
|
|
test('matches the canceling-statement message', () => {
|
|
expect(
|
|
isStatementTimeoutError(new Error('canceling statement due to statement timeout'))
|
|
).toBe(true);
|
|
});
|
|
|
|
test('does not match other errors', () => {
|
|
expect(isStatementTimeoutError(new Error('connection refused'))).toBe(false);
|
|
expect(isStatementTimeoutError(pgError('08006', 'connection_failure'))).toBe(false);
|
|
});
|
|
});
|
|
|
|
describe('isLockTimeoutError', () => {
|
|
test('matches SQLSTATE 55P03', () => {
|
|
expect(isLockTimeoutError(pgError('55P03', 'lock not available'))).toBe(true);
|
|
});
|
|
|
|
test('matches lock_not_available message', () => {
|
|
expect(isLockTimeoutError(new Error('could not obtain lock on row'))).toBe(true);
|
|
});
|
|
|
|
test('does not match statement timeouts', () => {
|
|
expect(isLockTimeoutError(pgError('57014', 'canceled'))).toBe(false);
|
|
});
|
|
});
|
|
|
|
describe('isRetryableConnError', () => {
|
|
test('matches Postgres class 08 codes', () => {
|
|
expect(isRetryableConnError(pgError('08000', 'connection_exception'))).toBe(true);
|
|
expect(isRetryableConnError(pgError('08003', 'connection_does_not_exist'))).toBe(true);
|
|
expect(isRetryableConnError(pgError('08006', 'connection_failure'))).toBe(true);
|
|
});
|
|
|
|
test('matches connection-refused message', () => {
|
|
expect(isRetryableConnError(new Error('connection refused'))).toBe(true);
|
|
});
|
|
|
|
test('matches ECONNRESET', () => {
|
|
expect(isRetryableConnError(new Error('ECONNRESET'))).toBe(true);
|
|
});
|
|
|
|
test('matches database-starting-up', () => {
|
|
expect(
|
|
isRetryableConnError(new Error('the database system is starting up'))
|
|
).toBe(true);
|
|
});
|
|
|
|
test('does NOT match statement timeouts', () => {
|
|
expect(isRetryableConnError(pgError('57014', 'canceled'))).toBe(false);
|
|
});
|
|
|
|
test('does NOT match lock timeouts', () => {
|
|
expect(isRetryableConnError(pgError('55P03', 'lock'))).toBe(false);
|
|
});
|
|
|
|
test('does not match arbitrary errors', () => {
|
|
expect(isRetryableConnError(new Error('something else'))).toBe(false);
|
|
});
|
|
|
|
// issue #1678: postgres.js's transaction-mode pooler reaps idle sockets and
|
|
// throws errors carrying `code: 'CONNECTION_ENDED'` (a library code, not an
|
|
// 08xxx SQLSTATE). Must be retryable via BOTH the code and the message form.
|
|
test('matches CONNECTION_ENDED via code', () => {
|
|
expect(isRetryableConnError(pgError('CONNECTION_ENDED', 'write CONNECTION_ENDED'))).toBe(true);
|
|
});
|
|
|
|
test('matches CONNECTION_ENDED via message even without the code', () => {
|
|
expect(isRetryableConnError(new Error('write CONNECTION_ENDED localhost:6543'))).toBe(true);
|
|
});
|
|
|
|
// The getter self-heal throws a GBrainError whose `problem` field is
|
|
// 'No database connection' — the existing typed-shape match must keep firing.
|
|
test('matches the instance-pool-reaped GBrainError shape (problem field)', () => {
|
|
const err = { problem: 'No database connection', message: 'instance pool torn down' };
|
|
expect(isRetryableConnError(err)).toBe(true);
|
|
});
|
|
|
|
// #1794: Supavisor session-pool exhaustion (EMAXCONNSESSION) + Postgres
|
|
// SQLSTATE 53300 too_many_connections. Transient under load — must retry so
|
|
// the resumable-sync checkpoint write survives the spike instead of being
|
|
// dropped (which is how #1794 lost 100% of progress).
|
|
test('matches EMAXCONNSESSION via message', () => {
|
|
expect(isRetryableConnError(new Error('EMAXCONNSESSION: max clients in session mode'))).toBe(true);
|
|
});
|
|
|
|
test('matches SQLSTATE 53300 too_many_connections via code', () => {
|
|
expect(isRetryableConnError(pgError('53300', 'too many connections for role'))).toBe(true);
|
|
});
|
|
|
|
test('matches "too many clients already" message', () => {
|
|
expect(isRetryableConnError(new Error('sorry, too many clients already'))).toBe(true);
|
|
});
|
|
|
|
test('matches reserved-slots message', () => {
|
|
expect(
|
|
isRetryableConnError(new Error('remaining connection slots are reserved for non-replication superuser connections'))
|
|
).toBe(true);
|
|
});
|
|
|
|
// 53300 must NOT accidentally widen to other 53xxx (e.g. 53400
|
|
// configuration_limit_exceeded is not a transient pool blip).
|
|
test('does NOT match unrelated 53xxx codes', () => {
|
|
expect(isRetryableConnError(pgError('53400', 'configuration limit exceeded'))).toBe(false);
|
|
});
|
|
});
|
|
|
|
describe('isRetryableError', () => {
|
|
test('union: returns true for conn AND statement-timeout', () => {
|
|
expect(isRetryableError(new Error('connection refused'))).toBe(true);
|
|
expect(isRetryableError(pgError('57014', 'canceled'))).toBe(true);
|
|
expect(isRetryableError(new Error('ECONNRESET'))).toBe(true);
|
|
});
|
|
|
|
test('still false for unrelated errors', () => {
|
|
expect(isRetryableError(new Error('foreign key violation'))).toBe(false);
|
|
});
|
|
});
|