mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
v0.42.22.0 fix(minions): supervisor progress watchdog + worker DB self-defense — alive-but-wedged worker self-heals (#1801) (#1824)
* fix(minions): supervisor progress watchdog + worker DB self-defense under supervision (#1801) Alive-but-wedged worker (dead DB pool, process still up) now self-heals in minutes instead of a silent 15h halt. - supervisor: progress watchdog restarts a child that makes no forward progress on claimable work (name+queue-scoped, active_healthy/due-delayed aware, startup-grace + loop-budget bounded); runtime handler-name derivation. - child-worker-supervisor: killChild gates on liveness not .killed (also fixes the existing shutdown SIGKILL no-op); restartCurrentChild kills the captured child ref; intentional restart doesn't count toward max_crashes. - worker: DB-liveness probe runs under supervision (db_dead self-exit), stall detection stays supervised-off. - doctor: standalone per-queue wedged_queue check + state->status fix in the remote queue_health check. - jobs/queue: queue-scoped getStats wedge fields + jobs stats WEDGED line. * fix(minions): wedge_restart_loop one-shot + supervised-probe comment + jobs-stats threshold (review) Pre-landing adversarial review findings: - wedge_restart_loop warn now fires once per exhausted window via a re-arming flag, not every health tick (was flooding the audit log for the full window). - Correct the stale GBRAIN_SUPERVISED comment: the DB probe runs under supervision now; only stall detection is skipped. - jobs stats WEDGED line reads GBRAIN_WEDGED_QUEUE_WARN_MINUTES so it agrees with the doctor wedged_queue threshold. * chore: bump version and changelog (v0.42.22.0) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * docs: queue-ops runbook + KEY_FILES for the #1801 wedge watchdog (v0.42.22.0) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
f3ade6c0c3
commit
f4959348c2
@@ -10,7 +10,7 @@
|
||||
*/
|
||||
|
||||
import { describe, it, expect, afterEach } from 'bun:test';
|
||||
import { chmodSync, mkdirSync, rmSync, writeFileSync } from 'fs';
|
||||
import { chmodSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
import {
|
||||
@@ -470,4 +470,143 @@ esac
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('issue #1801 — restartCurrentChild + killChild liveness fix', () => {
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
// ESRCH = no such process (dead). EPERM = process exists but we can't
|
||||
// signal it (alive) — under `bun test` a spawned child can land in a state
|
||||
// where kill(pid,0) reports EPERM, so treat anything-but-ESRCH as alive.
|
||||
const isAlive = (pid: number): boolean => {
|
||||
try { process.kill(pid, 0); return true; }
|
||||
catch (e) { return (e as NodeJS.ErrnoException)?.code !== 'ESRCH'; }
|
||||
};
|
||||
|
||||
// Worker that IGNORES SIGTERM and sleeps, so only SIGKILL can stop it.
|
||||
function makeSigtermIgnorer(name: string): Harness {
|
||||
return makeHarness(name, "trap '' TERM\nsleep 30");
|
||||
}
|
||||
|
||||
async function startInBackground(h: Harness): Promise<{
|
||||
sup: ChildWorkerSupervisor;
|
||||
events: ChildSupervisorEvent[];
|
||||
firstPid: number;
|
||||
stop: () => Promise<void>;
|
||||
}> {
|
||||
const events: ChildSupervisorEvent[] = [];
|
||||
let stopping = false;
|
||||
let resolveSpawn: (pid: number) => void;
|
||||
const firstSpawn = new Promise<number>((r) => { resolveSpawn = r; });
|
||||
const sup = new ChildWorkerSupervisor({
|
||||
cliPath: h.workerScript,
|
||||
args: [],
|
||||
maxCrashes: 100,
|
||||
_backoffFloorMs: 5,
|
||||
isStopping: () => stopping,
|
||||
onMaxCrashesExceeded: () => { stopping = true; },
|
||||
onEvent: (e) => {
|
||||
events.push(e);
|
||||
if (e.kind === 'worker_spawned') resolveSpawn(e.pid);
|
||||
},
|
||||
});
|
||||
const runPromise = sup.run();
|
||||
const firstPid = await firstSpawn;
|
||||
const stop = async () => {
|
||||
stopping = true;
|
||||
// SIGKILL-retry until run() returns (children ignore SIGTERM).
|
||||
for (let i = 0; i < 60; i++) {
|
||||
sup.killChild('SIGKILL');
|
||||
const done = await Promise.race([
|
||||
runPromise.then(() => true),
|
||||
sleep(50).then(() => false),
|
||||
]);
|
||||
if (done) return;
|
||||
}
|
||||
await runPromise;
|
||||
};
|
||||
return { sup, events, firstPid, stop };
|
||||
}
|
||||
|
||||
// Structural regression for Codex #1: killChild MUST gate on liveness
|
||||
// (exitCode/signalCode === null), NOT on `.killed`. `.killed` flips true the
|
||||
// moment a signal is *sent*, so a `!this._child.killed` guard makes a
|
||||
// follow-up SIGKILL (after an ignored SIGTERM) a silent no-op — the bug that
|
||||
// left the existing shutdown() drain unable to force-kill a stuck worker.
|
||||
// (The SIGTERM→SIGKILL behavior is exercised end-to-end by the
|
||||
// restartCurrentChild test below + standalone repros; a live-process
|
||||
// assertion that a SIGTERM-ignoring child survives is unreliable under the
|
||||
// `bun test` runtime, so the no-regression contract is pinned structurally.)
|
||||
it('killChild gates on liveness, not .killed (Codex #1 regression)', () => {
|
||||
const src = readFileSync(
|
||||
join(import.meta.dir, '..', 'src', 'core', 'minions', 'child-worker-supervisor.ts'),
|
||||
'utf8',
|
||||
);
|
||||
const killChildBody = src.slice(
|
||||
src.indexOf('killChild(signal: NodeJS.Signals)'),
|
||||
src.indexOf('awaitChildExit('),
|
||||
);
|
||||
// Strip comment lines so the doc note explaining the OLD bug (which names
|
||||
// `.killed`) doesn't trip the negative assertion — we check the CODE.
|
||||
const code = killChildBody
|
||||
.split('\n')
|
||||
.filter((l) => !l.trim().startsWith('//') && !l.trim().startsWith('*'))
|
||||
.join('\n');
|
||||
expect(code).toContain('exitCode === null');
|
||||
expect(code).toContain('signalCode === null');
|
||||
// The buggy `.killed` guard must be gone from the code.
|
||||
expect(code).not.toContain('.killed');
|
||||
});
|
||||
|
||||
it('restartCurrentChild SIGKILLs the captured child, respawns, labels wedge_restart, leaves crashCount=0', async () => {
|
||||
const h = makeSigtermIgnorer('restart-current');
|
||||
const ctx = await startInBackground(h);
|
||||
try {
|
||||
const oldPid = ctx.firstPid;
|
||||
await ctx.sup.restartCurrentChild(150); // SIGTERM ignored → SIGKILL after 150ms
|
||||
await sleep(400); // let the old child exit + run() respawn (ms:0 wedge backoff)
|
||||
|
||||
expect(isAlive(oldPid)).toBe(false); // captured child killed
|
||||
|
||||
const spawns = ctx.events.filter((e) => e.kind === 'worker_spawned');
|
||||
expect(spawns.length).toBeGreaterThanOrEqual(2); // respawned
|
||||
|
||||
const wedgeExit = ctx.events.find(
|
||||
(e) => e.kind === 'worker_exited' && e.likelyCause === 'wedge_restart',
|
||||
);
|
||||
expect(wedgeExit).toBeDefined();
|
||||
if (wedgeExit && wedgeExit.kind === 'worker_exited') {
|
||||
expect(wedgeExit.crashCount).toBe(0); // Codex #3 — not counted as a crash
|
||||
}
|
||||
|
||||
const wedgeBackoff = ctx.events.find(
|
||||
(e) => e.kind === 'backoff' && e.reason === 'wedge_restart',
|
||||
);
|
||||
expect(wedgeBackoff).toBeDefined();
|
||||
if (wedgeBackoff && wedgeBackoff.kind === 'backoff') {
|
||||
expect(wedgeBackoff.ms).toBe(0); // immediate respawn
|
||||
}
|
||||
|
||||
// Codex #2 — the respawned child is alive and was NOT killed by a stale
|
||||
// timer aimed at the old child.
|
||||
expect(ctx.sup.childAlive).toBe(true);
|
||||
} finally {
|
||||
await ctx.stop();
|
||||
h.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
it('repeated wedge restarts never trip max_crashes (crashCount stays 0)', async () => {
|
||||
const h = makeSigtermIgnorer('restart-no-crash');
|
||||
const ctx = await startInBackground(h);
|
||||
try {
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await ctx.sup.restartCurrentChild(120);
|
||||
await sleep(300);
|
||||
}
|
||||
expect(ctx.sup.crashCount).toBe(0); // three self-heals, zero crashes
|
||||
} finally {
|
||||
await ctx.stop();
|
||||
h.cleanup();
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
/**
|
||||
* issue #1801 fix #3 — doctor surfaces the wedged-queue signature as a health
|
||||
* ERROR (and the latent `state`→`status` regression in the remote queue_health
|
||||
* check is gone).
|
||||
*
|
||||
* computeWedgedQueueCheck is Postgres-only (short-circuits to ok on PGLite), so
|
||||
* we run its grouped SQL on a real PGLite engine behind a `kind: 'postgres'`
|
||||
* stub. Seeds verify: wedged → fail; live-lock → ok; null-completions →
|
||||
* conservative ok (the supervisor's startup-grace-aware watchdog owns that case);
|
||||
* per-queue grouping so a healthy queue doesn't mask a wedged one (Codex #15).
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
||||
import { readFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { computeWedgedQueueCheck } from '../src/commands/doctor.ts';
|
||||
import type { BrainEngine } from '../src/core/engine.ts';
|
||||
|
||||
let base: PGLiteEngine;
|
||||
let pgLike: BrainEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
base = new PGLiteEngine();
|
||||
await base.connect({});
|
||||
await base.initSchema();
|
||||
// computeWedgedQueueCheck only reads .kind + .executeRaw.
|
||||
pgLike = {
|
||||
kind: 'postgres',
|
||||
executeRaw: base.executeRaw.bind(base),
|
||||
} as unknown as BrainEngine;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await base.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
// Wipe only minion_jobs (preserve config/version that ensureSchema reads).
|
||||
await base.executeRaw('DELETE FROM minion_jobs');
|
||||
});
|
||||
|
||||
async function seed(
|
||||
queue: string,
|
||||
name: string,
|
||||
status: string,
|
||||
extra: { lockUntilSql?: string; updatedAtSql?: string } = {},
|
||||
): Promise<void> {
|
||||
await base.executeRaw(
|
||||
`INSERT INTO minion_jobs (name, queue, status, lock_until, updated_at)
|
||||
VALUES ($1, $2, $3, ${extra.lockUntilSql ?? 'NULL'}, ${extra.updatedAtSql ?? 'now()'})`,
|
||||
[name, queue, status],
|
||||
);
|
||||
}
|
||||
|
||||
describe('issue #1801 fix #3 — computeWedgedQueueCheck', () => {
|
||||
it('flags a wedged queue (waiting, 0 active_healthy, stale completion) as fail', async () => {
|
||||
await seed('default', 'cycle', 'waiting');
|
||||
await seed('default', 'cycle', 'completed', { updatedAtSql: "now() - interval '20 min'" });
|
||||
const check = await computeWedgedQueueCheck(pgLike);
|
||||
expect(check.status).toBe('fail');
|
||||
expect(check.message).toContain("'default'");
|
||||
});
|
||||
|
||||
it('does NOT flag when a job holds a live lock (active_healthy > 0)', async () => {
|
||||
await seed('default', 'cycle', 'waiting');
|
||||
await seed('default', 'cycle', 'active', { lockUntilSql: "now() + interval '5 min'" });
|
||||
await seed('default', 'cycle', 'completed', { updatedAtSql: "now() - interval '20 min'" });
|
||||
const check = await computeWedgedQueueCheck(pgLike);
|
||||
expect(check.status).toBe('ok');
|
||||
});
|
||||
|
||||
it('does NOT flag an expired-lock active row as healthy — still wedged (Codex #6)', async () => {
|
||||
await seed('default', 'cycle', 'waiting');
|
||||
await seed('default', 'cycle', 'active', { lockUntilSql: "now() - interval '2 min'" }); // expired
|
||||
await seed('default', 'cycle', 'completed', { updatedAtSql: "now() - interval '20 min'" });
|
||||
const check = await computeWedgedQueueCheck(pgLike);
|
||||
expect(check.status).toBe('fail'); // expired lock does not count as active_healthy
|
||||
});
|
||||
|
||||
it('is conservative on never-completed queues (null mins → ok)', async () => {
|
||||
await seed('default', 'cycle', 'waiting'); // no completed row → mins null
|
||||
const check = await computeWedgedQueueCheck(pgLike);
|
||||
expect(check.status).toBe('ok');
|
||||
});
|
||||
|
||||
it('groups by queue — a healthy queue does not mask a wedged one (Codex #15)', async () => {
|
||||
// healthy queue
|
||||
await seed('q-healthy', 'cycle', 'active', { lockUntilSql: "now() + interval '5 min'" });
|
||||
await seed('q-healthy', 'cycle', 'completed', { updatedAtSql: 'now()' });
|
||||
// wedged queue
|
||||
await seed('q-wedged', 'cycle', 'waiting');
|
||||
await seed('q-wedged', 'cycle', 'completed', { updatedAtSql: "now() - interval '30 min'" });
|
||||
const check = await computeWedgedQueueCheck(pgLike);
|
||||
expect(check.status).toBe('fail');
|
||||
expect(check.message).toContain("'q-wedged'");
|
||||
expect(check.message).not.toContain("'q-healthy'");
|
||||
});
|
||||
|
||||
it('returns ok on PGLite (no multi-process worker surface)', async () => {
|
||||
const check = await computeWedgedQueueCheck(base as unknown as BrainEngine);
|
||||
expect(check.status).toBe('ok');
|
||||
expect(check.message).toContain('PGLite');
|
||||
});
|
||||
});
|
||||
|
||||
describe('issue #1801 fix #3 — remote queue_health state→status regression', () => {
|
||||
it('doctor.ts no longer queries the non-existent `state` column', () => {
|
||||
const src = readFileSync(
|
||||
join(import.meta.dir, '..', 'src', 'commands', 'doctor.ts'),
|
||||
'utf8',
|
||||
);
|
||||
// The column is `status`; the pre-fix `WHERE state = 'active'` errored every
|
||||
// run and the catch silently returned "No queue activity".
|
||||
expect(src).not.toContain("state = 'active'");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,75 @@
|
||||
/**
|
||||
* issue #1801 fix #3 — MinionQueue.getStats() exposes a QUEUE-SCOPED wedge
|
||||
* block (the data behind the `jobs stats` WEDGED line). active_healthy counts
|
||||
* only live-lock active rows; the block is scoped to one queue so a healthy
|
||||
* worker on another queue can't mask a wedged one (Codex #14).
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { MinionQueue } from '../src/core/minions/queue.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
let queue: MinionQueue;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
queue = new MinionQueue(engine);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await engine.executeRaw('DELETE FROM minion_jobs');
|
||||
});
|
||||
|
||||
async function seed(
|
||||
q: string,
|
||||
name: string,
|
||||
status: string,
|
||||
extra: { lockUntilSql?: string; updatedAtSql?: string } = {},
|
||||
): Promise<void> {
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO minion_jobs (name, queue, status, lock_until, updated_at)
|
||||
VALUES ($1, $2, $3, ${extra.lockUntilSql ?? 'NULL'}, ${extra.updatedAtSql ?? 'now()'})`,
|
||||
[name, q, status],
|
||||
);
|
||||
}
|
||||
|
||||
describe('issue #1801 — getStats wedge block', () => {
|
||||
it('reports the wedge signature for the requested queue', async () => {
|
||||
await seed('default', 'cycle', 'waiting');
|
||||
await seed('default', 'cycle', 'active', { lockUntilSql: "now() - interval '1 min'" }); // expired
|
||||
await seed('default', 'cycle', 'completed', { updatedAtSql: "now() - interval '20 min'" });
|
||||
|
||||
const stats = await queue.getStats({ queue: 'default' });
|
||||
expect(stats.wedge.queue).toBe('default');
|
||||
expect(stats.wedge.active_healthy).toBe(0); // expired lock not counted
|
||||
expect(stats.wedge.waiting).toBe(1);
|
||||
expect(stats.wedge.minutes_since_completion).not.toBeNull();
|
||||
expect(stats.wedge.minutes_since_completion!).toBeGreaterThanOrEqual(15);
|
||||
});
|
||||
|
||||
it('counts a live-lock active row as healthy', async () => {
|
||||
await seed('default', 'cycle', 'active', { lockUntilSql: "now() + interval '5 min'" });
|
||||
const stats = await queue.getStats({ queue: 'default' });
|
||||
expect(stats.wedge.active_healthy).toBe(1);
|
||||
});
|
||||
|
||||
it('is queue-scoped — other queues do not bleed into the wedge block', async () => {
|
||||
await seed('other', 'cycle', 'waiting');
|
||||
const stats = await queue.getStats({ queue: 'default' });
|
||||
expect(stats.wedge.waiting).toBe(0); // nothing in 'default'
|
||||
});
|
||||
|
||||
it('defaults the wedge queue to "default" when unspecified', async () => {
|
||||
await seed('default', 'cycle', 'waiting');
|
||||
const stats = await queue.getStats();
|
||||
expect(stats.wedge.queue).toBe('default');
|
||||
expect(stats.wedge.waiting).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,250 @@
|
||||
/**
|
||||
* issue #1801 — supervisor progress watchdog.
|
||||
*
|
||||
* Two layers:
|
||||
* 1. Decision logic (counter / thresholds / startup grace / loop budget) via
|
||||
* the MinionSupervisor test seams + a stub engine — no DB, no spawn.
|
||||
* 2. SQL semantics of `queryWedgeSignals` against a real PGLite engine —
|
||||
* pins Codex #6 (expired-lock active row does NOT count as active_healthy),
|
||||
* #7 (due-delayed counts as claimable), #5 (name-scoping).
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
||||
import { MinionSupervisor, queryWedgeSignals } from '../src/core/minions/supervisor.ts';
|
||||
import type { BrainEngine } from '../src/core/engine.ts';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { resetPgliteState } from './helpers/reset-pglite.ts';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Layer 1: decision logic (stub engine + fake child supervisor)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface WedgeRow {
|
||||
stalled?: string;
|
||||
active_healthy?: string;
|
||||
waiting?: string;
|
||||
waiting_claimable?: string;
|
||||
last_completed?: string | null;
|
||||
last_completed_claimable?: string | null;
|
||||
}
|
||||
|
||||
function minutesAgoIso(min: number): string {
|
||||
return new Date(Date.now() - min * 60_000).toISOString();
|
||||
}
|
||||
|
||||
function makeSup(opts?: {
|
||||
row?: WedgeRow;
|
||||
childAlive?: boolean;
|
||||
inBackoff?: boolean;
|
||||
wedgeRestartMinutes?: number;
|
||||
wedgeRestartChecks?: number;
|
||||
wedgeRestartLoopBudget?: number;
|
||||
childStartedAtMsAgo?: number;
|
||||
}) {
|
||||
const events: Array<{ event: string; reason?: string; [k: string]: unknown }> = [];
|
||||
let restartCalls = 0;
|
||||
const rowRef: { current: WedgeRow } = {
|
||||
current: opts?.row ?? {},
|
||||
};
|
||||
|
||||
const stubEngine = {
|
||||
kind: 'postgres',
|
||||
async executeRaw() {
|
||||
const r = rowRef.current;
|
||||
return [{
|
||||
stalled: r.stalled ?? '0',
|
||||
active_healthy: r.active_healthy ?? '0',
|
||||
waiting: r.waiting ?? '0',
|
||||
waiting_claimable: r.waiting_claimable ?? '0',
|
||||
last_completed: r.last_completed ?? null,
|
||||
last_completed_claimable: r.last_completed_claimable ?? null,
|
||||
}];
|
||||
},
|
||||
} as unknown as BrainEngine;
|
||||
|
||||
const sup = new MinionSupervisor(stubEngine, {
|
||||
cliPath: '/bin/true',
|
||||
maxRssMb: 0, // skip cgroup auto-size
|
||||
healthInterval: 60_000,
|
||||
wedgeRestartMinutes: opts?.wedgeRestartMinutes ?? 15,
|
||||
wedgeRestartChecks: opts?.wedgeRestartChecks ?? 3,
|
||||
wedgeRestartLoopBudget: opts?.wedgeRestartLoopBudget ?? 3,
|
||||
startupGraceMs: 120_000,
|
||||
onEvent: (e) => events.push(e as { event: string; reason?: string }),
|
||||
});
|
||||
|
||||
sup._setChildSupervisorForTests({
|
||||
childAlive: opts?.childAlive ?? true,
|
||||
inBackoff: opts?.inBackoff ?? false,
|
||||
restartCurrentChild: async () => { restartCalls++; },
|
||||
} as never);
|
||||
|
||||
sup._setWedgeStateForTests({
|
||||
handlerNames: ['cycle'],
|
||||
childStartedAt: Date.now() - (opts?.childStartedAtMsAgo ?? 600_000), // 10 min ago
|
||||
});
|
||||
|
||||
return {
|
||||
sup,
|
||||
events,
|
||||
getRestartCalls: () => restartCalls,
|
||||
setRow: (r: WedgeRow) => { rowRef.current = r; },
|
||||
};
|
||||
}
|
||||
|
||||
const WEDGE_ROW: WedgeRow = {
|
||||
active_healthy: '0',
|
||||
waiting: '0',
|
||||
waiting_claimable: '5',
|
||||
last_completed_claimable: minutesAgoIso(20), // stale > 15
|
||||
};
|
||||
|
||||
describe('issue #1801 — supervisor wedge decision logic', () => {
|
||||
it('restarts a wedged worker after wedgeRestartChecks consecutive checks', async () => {
|
||||
const h = makeSup({ row: WEDGE_ROW });
|
||||
await h.sup._healthCheckOnceForTests();
|
||||
await h.sup._healthCheckOnceForTests();
|
||||
expect(h.getRestartCalls()).toBe(0); // not yet (default 3 checks)
|
||||
await h.sup._healthCheckOnceForTests();
|
||||
expect(h.getRestartCalls()).toBe(1);
|
||||
expect(h.events.some((e) => e.event === 'health_warn' && e.reason === 'restarting_wedged_worker')).toBe(true);
|
||||
});
|
||||
|
||||
it('never escalates when a job holds a live lock (active_healthy > 0)', async () => {
|
||||
const h = makeSup({ row: { ...WEDGE_ROW, active_healthy: '2' } });
|
||||
for (let i = 0; i < 5; i++) await h.sup._healthCheckOnceForTests();
|
||||
expect(h.getRestartCalls()).toBe(0);
|
||||
});
|
||||
|
||||
it('suppresses restart during the startup grace window (Codex #9/#10)', async () => {
|
||||
const h = makeSup({ row: WEDGE_ROW, childStartedAtMsAgo: 1_000 }); // just spawned
|
||||
for (let i = 0; i < 4; i++) await h.sup._healthCheckOnceForTests();
|
||||
expect(h.getRestartCalls()).toBe(0);
|
||||
});
|
||||
|
||||
it('wedgeRestartMinutes <= 0 disables the watchdog', async () => {
|
||||
const h = makeSup({ row: WEDGE_ROW, wedgeRestartMinutes: 0 });
|
||||
for (let i = 0; i < 4; i++) await h.sup._healthCheckOnceForTests();
|
||||
expect(h.getRestartCalls()).toBe(0);
|
||||
});
|
||||
|
||||
it('inBackoff suppresses escalation (respawn window)', async () => {
|
||||
const h = makeSup({ row: WEDGE_ROW, inBackoff: true });
|
||||
for (let i = 0; i < 4; i++) await h.sup._healthCheckOnceForTests();
|
||||
expect(h.getRestartCalls()).toBe(0);
|
||||
});
|
||||
|
||||
it('resets the counter when the wedge clears', async () => {
|
||||
const h = makeSup({ row: WEDGE_ROW, wedgeRestartChecks: 3 });
|
||||
await h.sup._healthCheckOnceForTests(); // wedged 1
|
||||
await h.sup._healthCheckOnceForTests(); // wedged 2
|
||||
h.setRow({ ...WEDGE_ROW, active_healthy: '1' }); // healthy → reset
|
||||
await h.sup._healthCheckOnceForTests();
|
||||
expect(h.sup._consecutiveWedgedChecksForTests).toBe(0);
|
||||
h.setRow(WEDGE_ROW); // wedged again
|
||||
await h.sup._healthCheckOnceForTests(); // 1
|
||||
await h.sup._healthCheckOnceForTests(); // 2
|
||||
expect(h.getRestartCalls()).toBe(0); // still under 3 after the reset
|
||||
await h.sup._healthCheckOnceForTests(); // 3
|
||||
expect(h.getRestartCalls()).toBe(1);
|
||||
});
|
||||
|
||||
it('loop budget (>=) stops restarting and emits wedge_restart_loop ONCE (Codex #13 + loop-spam fix)', async () => {
|
||||
const h = makeSup({ row: WEDGE_ROW, wedgeRestartChecks: 1, wedgeRestartLoopBudget: 2 });
|
||||
await h.sup._healthCheckOnceForTests(); // restart 1 (timestamps: 0 < 2 → push)
|
||||
await h.sup._healthCheckOnceForTests(); // restart 2 (timestamps: 1 < 2 → push)
|
||||
await h.sup._healthCheckOnceForTests(); // budget: 2 >= 2 → no restart, loop warn
|
||||
await h.sup._healthCheckOnceForTests(); // still exhausted → MUST NOT re-warn
|
||||
await h.sup._healthCheckOnceForTests(); // still exhausted → MUST NOT re-warn
|
||||
expect(h.getRestartCalls()).toBe(2);
|
||||
const loopWarns = h.events.filter((e) => e.event === 'health_warn' && e.reason === 'wedge_restart_loop');
|
||||
expect(loopWarns.length).toBe(1); // fired once on entry, not every tick (no audit flood)
|
||||
});
|
||||
|
||||
it('null last_completed_claimable wedges only after the startup grace', async () => {
|
||||
// Never-completed claimable work, child past grace → treated as stale.
|
||||
const h = makeSup({
|
||||
row: { active_healthy: '0', waiting_claimable: '3', last_completed_claimable: null },
|
||||
wedgeRestartChecks: 3,
|
||||
});
|
||||
for (let i = 0; i < 3; i++) await h.sup._healthCheckOnceForTests();
|
||||
expect(h.getRestartCalls()).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Layer 2: SQL semantics of queryWedgeSignals (real PGLite)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await resetPgliteState(engine);
|
||||
});
|
||||
|
||||
async function seed(
|
||||
queue: string,
|
||||
name: string,
|
||||
status: string,
|
||||
extra: { lockUntilSql?: string; delayUntilSql?: string; updatedAtSql?: string } = {},
|
||||
): Promise<void> {
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO minion_jobs (name, queue, status, lock_until, delay_until, updated_at)
|
||||
VALUES ($1, $2, $3, ${extra.lockUntilSql ?? 'NULL'}, ${extra.delayUntilSql ?? 'NULL'}, ${extra.updatedAtSql ?? 'now()'})`,
|
||||
[name, queue, status],
|
||||
);
|
||||
}
|
||||
|
||||
describe('issue #1801 — queryWedgeSignals SQL semantics (PGLite)', () => {
|
||||
it('scopes claimable by name, counts due-delayed, excludes not-due-delayed (Codex #5/#7)', async () => {
|
||||
const q = 'sql-a';
|
||||
await seed(q, 'cycle', 'waiting'); // claimable
|
||||
await seed(q, 'cycle', 'delayed', { delayUntilSql: "now() - interval '1 min'" }); // due → claimable
|
||||
await seed(q, 'cycle', 'delayed', { delayUntilSql: "now() + interval '1 hour'" }); // not due → not
|
||||
await seed(q, 'other', 'waiting'); // wrong name → not claimable
|
||||
await seed(q, 'cycle', 'completed', { updatedAtSql: 'now()' });
|
||||
|
||||
const sig = await queryWedgeSignals(engine, q, ['cycle']);
|
||||
expect(sig.waitingClaimable).toBe(2); // waiting + due-delayed
|
||||
expect(sig.waiting).toBe(2); // total waiting (cycle + other)
|
||||
expect(sig.lastCompletedClaimable).not.toBeNull();
|
||||
});
|
||||
|
||||
it('expired-lock active row counts as stalled, NOT active_healthy (Codex #6)', async () => {
|
||||
const q = 'sql-b';
|
||||
await seed(q, 'cycle', 'active', { lockUntilSql: "now() - interval '1 min'" }); // expired
|
||||
await seed(q, 'cycle', 'waiting');
|
||||
|
||||
const sig = await queryWedgeSignals(engine, q, ['cycle']);
|
||||
expect(sig.activeHealthy).toBe(0); // expired lock does NOT suppress the wedge
|
||||
expect(sig.stalled).toBe(1);
|
||||
expect(sig.waitingClaimable).toBe(1);
|
||||
});
|
||||
|
||||
it('live-lock active row counts as active_healthy', async () => {
|
||||
const q = 'sql-c';
|
||||
await seed(q, 'cycle', 'active', { lockUntilSql: "now() + interval '5 min'" }); // live
|
||||
await seed(q, 'cycle', 'waiting');
|
||||
|
||||
const sig = await queryWedgeSignals(engine, q, ['cycle']);
|
||||
expect(sig.activeHealthy).toBe(1);
|
||||
expect(sig.stalled).toBe(0);
|
||||
});
|
||||
|
||||
it('is queue-scoped — other queues do not bleed in', async () => {
|
||||
await seed('q1', 'cycle', 'waiting');
|
||||
await seed('q2', 'cycle', 'waiting');
|
||||
const sig = await queryWedgeSignals(engine, 'q1', ['cycle']);
|
||||
expect(sig.waitingClaimable).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,109 @@
|
||||
/**
|
||||
* issue #1801 fix #2 — the worker's DB-liveness probe runs EVEN under a
|
||||
* supervisor (GBRAIN_SUPERVISED=1), so a supervised worker whose own pool dies
|
||||
* self-exits (db_dead → CLI process.exit(1) → supervisor respawns a fresh pool)
|
||||
* instead of sitting alive-but-wedged forever. Stall detection STAYS gated to
|
||||
* non-supervised (the supervisor's progress watchdog owns forward-progress).
|
||||
*
|
||||
* Behavioral: real PGLite engine with `executeRaw('SELECT 1')` monkeypatched to
|
||||
* throw, so the probe fails and emitUnhealthy('db_dead') fires after
|
||||
* dbFailExitAfter probes. Structural: pins the gating split in worker.ts so a
|
||||
* refactor can't silently re-disable the probe under supervision.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { readFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { MinionWorker, type UnhealthyReason } from '../src/core/minions/worker.ts';
|
||||
import { withEnv } from './helpers/with-env.ts';
|
||||
|
||||
// No resetPgliteState/beforeEach: these tests never insert jobs (empty queue),
|
||||
// and resetPgliteState TRUNCATEs the `config` table that carries the minion
|
||||
// schema `version` key the worker's ensureSchema() reads.
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
/** Make the DB-liveness probe (`SELECT 1`) throw; delegate every other query.
|
||||
* Returns a restore fn that removes the instance override (falls back to the
|
||||
* prototype method). */
|
||||
function breakLivenessProbe(eng: PGLiteEngine): () => void {
|
||||
const real = eng.executeRaw.bind(eng);
|
||||
(eng as { executeRaw: unknown }).executeRaw = async (sql: string, params?: unknown[]) => {
|
||||
if (typeof sql === 'string' && sql.trim() === 'SELECT 1') {
|
||||
throw new Error('probe boom (simulated dead pool)');
|
||||
}
|
||||
return real(sql, params as never);
|
||||
};
|
||||
return () => { delete (eng as { executeRaw?: unknown }).executeRaw; };
|
||||
}
|
||||
|
||||
async function runUntilUnhealthy(supervised: boolean): Promise<UnhealthyReason | null> {
|
||||
const restore = breakLivenessProbe(engine);
|
||||
try {
|
||||
return await withEnv(
|
||||
supervised ? { GBRAIN_SUPERVISED: '1' } : { GBRAIN_SUPERVISED: undefined },
|
||||
async () => {
|
||||
const worker = new MinionWorker(engine, {
|
||||
queue: 'default',
|
||||
concurrency: 1,
|
||||
pollInterval: 25,
|
||||
maxRssMb: 0,
|
||||
healthCheckInterval: 20,
|
||||
dbFailExitAfter: 3,
|
||||
dbProbeTimeoutMs: 200,
|
||||
});
|
||||
worker.register('noop', async () => {});
|
||||
|
||||
const got = new Promise<UnhealthyReason>((resolve) => {
|
||||
worker.on('unhealthy', (i) => resolve(i));
|
||||
});
|
||||
const runPromise = worker.start();
|
||||
const captured = await Promise.race([
|
||||
got,
|
||||
new Promise<null>((r) => setTimeout(() => r(null), 3000)),
|
||||
]);
|
||||
worker.stop();
|
||||
await runPromise;
|
||||
return captured;
|
||||
},
|
||||
);
|
||||
} finally {
|
||||
restore();
|
||||
}
|
||||
}
|
||||
|
||||
describe('issue #1801 fix #2 — supervised DB self-defense', () => {
|
||||
it('a SUPERVISED worker with a dead pool emits db_dead (probe runs under supervision)', async () => {
|
||||
const info = await runUntilUnhealthy(true);
|
||||
expect(info).not.toBeNull();
|
||||
expect(info?.reason).toBe('db_dead');
|
||||
}, 10_000);
|
||||
|
||||
it('an UNSUPERVISED worker with a dead pool also emits db_dead (back-compat)', async () => {
|
||||
const info = await runUntilUnhealthy(false);
|
||||
expect(info).not.toBeNull();
|
||||
expect(info?.reason).toBe('db_dead');
|
||||
}, 10_000);
|
||||
|
||||
it('structural: DB probe is NOT gated on !isSupervisedChild; stall detection IS', () => {
|
||||
const src = readFileSync(
|
||||
join(import.meta.dir, '..', 'src', 'core', 'minions', 'worker.ts'),
|
||||
'utf8',
|
||||
);
|
||||
// Outer self-health guard must NOT require non-supervised anymore.
|
||||
expect(src).toContain('if (this.opts.healthCheckInterval > 0) {');
|
||||
expect(src).not.toContain('if (!isSupervisedChild && this.opts.healthCheckInterval > 0)');
|
||||
// Stall detection must still be wrapped in a non-supervised guard.
|
||||
expect(src).toMatch(/Stall detection \(NON-supervised only\)[\s\S]*?if \(!isSupervisedChild\)/);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user