mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
f4959348c2
* fix(minions): supervisor progress watchdog + worker DB self-defense under supervision (#1801) Alive-but-wedged worker (dead DB pool, process still up) now self-heals in minutes instead of a silent 15h halt. - supervisor: progress watchdog restarts a child that makes no forward progress on claimable work (name+queue-scoped, active_healthy/due-delayed aware, startup-grace + loop-budget bounded); runtime handler-name derivation. - child-worker-supervisor: killChild gates on liveness not .killed (also fixes the existing shutdown SIGKILL no-op); restartCurrentChild kills the captured child ref; intentional restart doesn't count toward max_crashes. - worker: DB-liveness probe runs under supervision (db_dead self-exit), stall detection stays supervised-off. - doctor: standalone per-queue wedged_queue check + state->status fix in the remote queue_health check. - jobs/queue: queue-scoped getStats wedge fields + jobs stats WEDGED line. * fix(minions): wedge_restart_loop one-shot + supervised-probe comment + jobs-stats threshold (review) Pre-landing adversarial review findings: - wedge_restart_loop warn now fires once per exhausted window via a re-arming flag, not every health tick (was flooding the audit log for the full window). - Correct the stale GBRAIN_SUPERVISED comment: the DB probe runs under supervision now; only stall detection is skipped. - jobs stats WEDGED line reads GBRAIN_WEDGED_QUEUE_WARN_MINUTES so it agrees with the doctor wedged_queue threshold. * chore: bump version and changelog (v0.42.22.0) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * docs: queue-ops runbook + KEY_FILES for the #1801 wedge watchdog (v0.42.22.0) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
76 lines
2.7 KiB
TypeScript
76 lines
2.7 KiB
TypeScript
/**
|
|
* issue #1801 fix #3 — MinionQueue.getStats() exposes a QUEUE-SCOPED wedge
|
|
* block (the data behind the `jobs stats` WEDGED line). active_healthy counts
|
|
* only live-lock active rows; the block is scoped to one queue so a healthy
|
|
* worker on another queue can't mask a wedged one (Codex #14).
|
|
*/
|
|
|
|
import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
|
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
|
import { MinionQueue } from '../src/core/minions/queue.ts';
|
|
|
|
let engine: PGLiteEngine;
|
|
let queue: MinionQueue;
|
|
|
|
beforeAll(async () => {
|
|
engine = new PGLiteEngine();
|
|
await engine.connect({});
|
|
await engine.initSchema();
|
|
queue = new MinionQueue(engine);
|
|
});
|
|
|
|
afterAll(async () => {
|
|
await engine.disconnect();
|
|
});
|
|
|
|
beforeEach(async () => {
|
|
await engine.executeRaw('DELETE FROM minion_jobs');
|
|
});
|
|
|
|
async function seed(
|
|
q: string,
|
|
name: string,
|
|
status: string,
|
|
extra: { lockUntilSql?: string; updatedAtSql?: string } = {},
|
|
): Promise<void> {
|
|
await engine.executeRaw(
|
|
`INSERT INTO minion_jobs (name, queue, status, lock_until, updated_at)
|
|
VALUES ($1, $2, $3, ${extra.lockUntilSql ?? 'NULL'}, ${extra.updatedAtSql ?? 'now()'})`,
|
|
[name, q, status],
|
|
);
|
|
}
|
|
|
|
describe('issue #1801 — getStats wedge block', () => {
|
|
it('reports the wedge signature for the requested queue', async () => {
|
|
await seed('default', 'cycle', 'waiting');
|
|
await seed('default', 'cycle', 'active', { lockUntilSql: "now() - interval '1 min'" }); // expired
|
|
await seed('default', 'cycle', 'completed', { updatedAtSql: "now() - interval '20 min'" });
|
|
|
|
const stats = await queue.getStats({ queue: 'default' });
|
|
expect(stats.wedge.queue).toBe('default');
|
|
expect(stats.wedge.active_healthy).toBe(0); // expired lock not counted
|
|
expect(stats.wedge.waiting).toBe(1);
|
|
expect(stats.wedge.minutes_since_completion).not.toBeNull();
|
|
expect(stats.wedge.minutes_since_completion!).toBeGreaterThanOrEqual(15);
|
|
});
|
|
|
|
it('counts a live-lock active row as healthy', async () => {
|
|
await seed('default', 'cycle', 'active', { lockUntilSql: "now() + interval '5 min'" });
|
|
const stats = await queue.getStats({ queue: 'default' });
|
|
expect(stats.wedge.active_healthy).toBe(1);
|
|
});
|
|
|
|
it('is queue-scoped — other queues do not bleed into the wedge block', async () => {
|
|
await seed('other', 'cycle', 'waiting');
|
|
const stats = await queue.getStats({ queue: 'default' });
|
|
expect(stats.wedge.waiting).toBe(0); // nothing in 'default'
|
|
});
|
|
|
|
it('defaults the wedge queue to "default" when unspecified', async () => {
|
|
await seed('default', 'cycle', 'waiting');
|
|
const stats = await queue.getStats();
|
|
expect(stats.wedge.queue).toBe('default');
|
|
expect(stats.wedge.waiting).toBe(1);
|
|
});
|
|
});
|