mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-31 04:07:52 +00:00
fix(jobs): reap stale dead-holder cycle/sync locks (#1972)
A crashed sync (OOM, recycle, SIGKILL) stranded its gbrain_cycle_locks row until something contended for it — reclaim was on-contention only. Add a host-scoped background reaper: reapDeadHolderLocks deletes locks whose holder PID is provably dead on this host, scoped to the gbrain-sync:*/gbrain-cycle* namespaces only (never elections/supervisor/reindex), with a snapshot-matched delete (date_trunc on acquired_at) that is TOCTOU-safe against PID reuse. Reuses isHolderDeadLocally (same-host + ESRCH + 60s grace). doctor --fix now auto-reaps for no-autopilot brains. DRY: selectLockRows + shared mapper now back inspectLock + listStaleLocks (killed the triplication).
This commit is contained in:
+36
-5
@@ -2346,12 +2346,39 @@ export function checkAutopilotLockScope(): Check {
|
||||
* but the main work is blocked. Requires explicit heartbeat probe;
|
||||
* speculation until production data shows the case.
|
||||
*/
|
||||
export async function checkStaleLocks(engine: BrainEngine): Promise<Check> {
|
||||
export async function checkStaleLocks(
|
||||
engine: BrainEngine,
|
||||
opts: { fix?: boolean; dryRun?: boolean } = {},
|
||||
): Promise<Check> {
|
||||
try {
|
||||
const { listStaleLocks } = await import('../core/db-lock.ts');
|
||||
const { listStaleLocks, reapDeadHolderLocks } = await import('../core/db-lock.ts');
|
||||
|
||||
// #1972: under `gbrain doctor --fix`, reap dead-holder sync/cycle locks
|
||||
// using the SAME namespace-scoped, host-scoped, snapshot-matched reaper the
|
||||
// cycle runs at start. This is the self-heal path for no-autopilot brains: a
|
||||
// brain that never runs `gbrain dream` never hits the cycle-start sweep, so
|
||||
// doctor --fix is how its crashed-sync locks get cleared. DB-only, so it's
|
||||
// orthogonal to (and unaffected by) the skills-dir --fix safety gate above.
|
||||
// Best-effort: a reap failure falls through to the warn path below.
|
||||
let reapedIds: string[] = [];
|
||||
if (opts.fix && !opts.dryRun) {
|
||||
try {
|
||||
reapedIds = (await reapDeadHolderLocks(engine)).reapedIds;
|
||||
} catch { /* fall through; listStaleLocks still surfaces remaining locks */ }
|
||||
}
|
||||
const reapedNote = reapedIds.length > 0
|
||||
? `Reaped ${reapedIds.length} dead-holder lock(s): ${reapedIds.join(', ')}.`
|
||||
: null;
|
||||
|
||||
const stale = await listStaleLocks(engine);
|
||||
if (stale.length === 0) {
|
||||
return { name: 'stale_locks', status: 'ok', message: 'No stale locks (no rows with ttl_expires_at < NOW())' };
|
||||
return {
|
||||
name: 'stale_locks',
|
||||
status: 'ok',
|
||||
message: reapedNote
|
||||
? `${reapedNote} No stale locks remain.`
|
||||
: 'No stale locks (no rows with ttl_expires_at < NOW())',
|
||||
};
|
||||
}
|
||||
const lines = stale.slice(0, 10).map(s => {
|
||||
const ageH = Math.floor(s.age_ms / 3600_000);
|
||||
@@ -2360,11 +2387,15 @@ export async function checkStaleLocks(engine: BrainEngine): Promise<Check> {
|
||||
return ` ${s.id} (pid ${s.holder_pid} on ${s.holder_host}, age ${ageH}h) → ${breakHint}`;
|
||||
});
|
||||
const tail = stale.length > 10 ? ` ... and ${stale.length - 10} more.` : null;
|
||||
const header = opts.fix
|
||||
? `${stale.length} stale lock(s) remain that could not be auto-reaped (live holder, cross-host, or within the PID-reuse grace):`
|
||||
: `${stale.length} stale lock(s) detected (ttl_expires_at < NOW()):`;
|
||||
return {
|
||||
name: 'stale_locks',
|
||||
status: 'warn',
|
||||
message: [
|
||||
`${stale.length} stale lock(s) detected (ttl_expires_at < NOW()):`,
|
||||
reapedNote,
|
||||
header,
|
||||
...lines,
|
||||
tail,
|
||||
].filter(Boolean).join('\n'),
|
||||
@@ -6951,7 +6982,7 @@ export async function buildChecks(
|
||||
checks.push(checkAutopilotLockScope());
|
||||
// v0.41.6.0 D3 — stale_locks (gbrain_cycle_locks rows with ttl_expires_at < NOW())
|
||||
progress.heartbeat('stale_locks');
|
||||
checks.push(await checkStaleLocks(engine));
|
||||
checks.push(await checkStaleLocks(engine, { fix: doFix, dryRun }));
|
||||
// v0.38 — cycle_phase_scope (informational; no DB cost)
|
||||
progress.heartbeat('cycle_phase_scope');
|
||||
checks.push(checkCyclePhaseScope());
|
||||
|
||||
+182
-94
@@ -307,58 +307,37 @@ export interface LockSnapshot {
|
||||
ms_since_last_refresh: number | null;
|
||||
}
|
||||
|
||||
export async function inspectLock(engine: BrainEngine, lockId: string): Promise<LockSnapshot | null> {
|
||||
const maybePG = engine as unknown as { sql?: (...args: unknown[]) => Promise<unknown> };
|
||||
const maybePGLite = engine as unknown as {
|
||||
db?: { query: (sql: string, params?: unknown[]) => Promise<{ rows: unknown[] }> };
|
||||
};
|
||||
/**
|
||||
* Raw row shape returned by every `gbrain_cycle_locks` SELECT. Lives here so
|
||||
* `selectLockRows` (the single canonical reader) and its mapper share one type.
|
||||
*/
|
||||
interface RawLockRow {
|
||||
id?: string;
|
||||
holder_pid?: number;
|
||||
holder_host?: string;
|
||||
acquired_at?: Date | string;
|
||||
ttl_expires_at?: Date | string;
|
||||
last_refreshed_at?: Date | string | null;
|
||||
}
|
||||
|
||||
let row: {
|
||||
id?: string;
|
||||
holder_pid?: number;
|
||||
holder_host?: string;
|
||||
acquired_at?: Date | string;
|
||||
ttl_expires_at?: Date | string;
|
||||
last_refreshed_at?: Date | string | null;
|
||||
} | undefined;
|
||||
|
||||
if (engine.kind === 'postgres' && maybePG.sql) {
|
||||
const sql = maybePG.sql as any;
|
||||
const rows = await sql`
|
||||
SELECT id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at
|
||||
FROM gbrain_cycle_locks
|
||||
WHERE id = ${lockId}
|
||||
`;
|
||||
row = rows[0];
|
||||
} else if (engine.kind === 'pglite' && maybePGLite.db) {
|
||||
const { rows } = await maybePGLite.db.query(
|
||||
`SELECT id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at
|
||||
FROM gbrain_cycle_locks
|
||||
WHERE id = $1`,
|
||||
[lockId],
|
||||
);
|
||||
row = rows[0] as typeof row;
|
||||
} else {
|
||||
throw new Error(`Unknown engine kind for inspectLock: ${engine.kind}`);
|
||||
}
|
||||
|
||||
if (!row || row.holder_pid === undefined || !row.acquired_at || !row.ttl_expires_at) return null;
|
||||
/** Canonical column list for every lock SELECT — keep in lockstep with `RawLockRow`. */
|
||||
const LOCK_SELECT_COLS = 'id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at';
|
||||
|
||||
/**
|
||||
* Row → `LockSnapshot` mapper. Coerces postgres.js Date|string columns to Date
|
||||
* and computes the derived fields (age_ms, ttl_expired, ms_since_last_refresh).
|
||||
* Returns null for a structurally-incomplete row (missing pid / acquired_at /
|
||||
* ttl). v0.41.13.0: last_refreshed_at may be NULL on pre-v98 brains.
|
||||
*/
|
||||
function rowToLockSnapshot(row: RawLockRow, now: number): LockSnapshot | null {
|
||||
if (row.holder_pid === undefined || !row.acquired_at || !row.ttl_expires_at) return null;
|
||||
const acquired = row.acquired_at instanceof Date ? row.acquired_at : new Date(row.acquired_at);
|
||||
const ttlExpires = row.ttl_expires_at instanceof Date ? row.ttl_expires_at : new Date(row.ttl_expires_at);
|
||||
const now = Date.now();
|
||||
// v0.41.13.0: last_refreshed_at may be NULL on pre-v98 brains that have
|
||||
// the column but no acquire has happened since the migration ran. Render
|
||||
// both `last_refreshed_at` and the computed delta as null so callers can
|
||||
// distinguish "never observed a refresh" from "refresh fired N ms ago".
|
||||
const lastRefreshed = row.last_refreshed_at == null
|
||||
? null
|
||||
: (row.last_refreshed_at instanceof Date
|
||||
? row.last_refreshed_at
|
||||
: new Date(row.last_refreshed_at));
|
||||
|
||||
: (row.last_refreshed_at instanceof Date ? row.last_refreshed_at : new Date(row.last_refreshed_at));
|
||||
return {
|
||||
id: lockId,
|
||||
id: String(row.id ?? ''),
|
||||
holder_pid: Number(row.holder_pid),
|
||||
holder_host: String(row.holder_host ?? ''),
|
||||
acquired_at: acquired,
|
||||
@@ -370,6 +349,59 @@ export async function inspectLock(engine: BrainEngine, lockId: string): Promise<
|
||||
};
|
||||
}
|
||||
|
||||
interface SelectLockRowsOpts {
|
||||
/** Return only the row for this exact lock id (the `inspectLock` case). */
|
||||
lockId?: string;
|
||||
/** Return only rows whose ttl_expires_at < NOW() (the `listStaleLocks` case). */
|
||||
staleOnly?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* The single canonical reader for `gbrain_cycle_locks`. Engine-branches once
|
||||
* (postgres / pglite) so `inspectLock`, `listStaleLocks`, and the reaper don't
|
||||
* each re-roll the SELECT + Date-coercion (previously triplicated). Returns
|
||||
* fully-mapped `LockSnapshot[]`; callers filter in JS (the table holds 0-2 rows
|
||||
* in practice, so a no-WHERE full read is cheap and keeps the SQL trivial).
|
||||
*/
|
||||
async function selectLockRows(engine: BrainEngine, opts: SelectLockRowsOpts = {}): Promise<LockSnapshot[]> {
|
||||
const maybePG = engine as unknown as { sql?: (...args: unknown[]) => Promise<unknown> };
|
||||
const maybePGLite = engine as unknown as {
|
||||
db?: { query: (sql: string, params?: unknown[]) => Promise<{ rows: unknown[] }> };
|
||||
};
|
||||
|
||||
let rows: RawLockRow[];
|
||||
if (engine.kind === 'postgres' && maybePG.sql) {
|
||||
const sql = maybePG.sql as any;
|
||||
if (opts.lockId !== undefined) {
|
||||
rows = await sql`SELECT id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at FROM gbrain_cycle_locks WHERE id = ${opts.lockId}`;
|
||||
} else if (opts.staleOnly) {
|
||||
rows = await sql`SELECT id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at FROM gbrain_cycle_locks WHERE ttl_expires_at < NOW() ORDER BY acquired_at`;
|
||||
} else {
|
||||
rows = await sql`SELECT id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at FROM gbrain_cycle_locks ORDER BY acquired_at`;
|
||||
}
|
||||
} else if (engine.kind === 'pglite' && maybePGLite.db) {
|
||||
if (opts.lockId !== undefined) {
|
||||
rows = (await maybePGLite.db.query(`SELECT ${LOCK_SELECT_COLS} FROM gbrain_cycle_locks WHERE id = $1`, [opts.lockId])).rows as RawLockRow[];
|
||||
} else if (opts.staleOnly) {
|
||||
rows = (await maybePGLite.db.query(`SELECT ${LOCK_SELECT_COLS} FROM gbrain_cycle_locks WHERE ttl_expires_at < NOW() ORDER BY acquired_at`)).rows as RawLockRow[];
|
||||
} else {
|
||||
rows = (await maybePGLite.db.query(`SELECT ${LOCK_SELECT_COLS} FROM gbrain_cycle_locks ORDER BY acquired_at`)).rows as RawLockRow[];
|
||||
}
|
||||
} else {
|
||||
throw new Error(`Unknown engine kind for selectLockRows: ${engine.kind}`);
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
return rows
|
||||
.map((r) => rowToLockSnapshot(r, now))
|
||||
.filter((s): s is LockSnapshot => s !== null);
|
||||
}
|
||||
|
||||
export async function inspectLock(engine: BrainEngine, lockId: string): Promise<LockSnapshot | null> {
|
||||
const rows = await selectLockRows(engine, { lockId });
|
||||
return rows[0] ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.41.6.0 D3: list every lock whose TTL has expired. Used by gbrain
|
||||
* doctor's `stale_locks` check. The query reuses the same canonical
|
||||
@@ -377,55 +409,7 @@ export async function inspectLock(engine: BrainEngine, lockId: string): Promise<
|
||||
* UPDATE-on-conflict already trusts — no parallel heuristic.
|
||||
*/
|
||||
export async function listStaleLocks(engine: BrainEngine): Promise<LockSnapshot[]> {
|
||||
const maybePG = engine as unknown as { sql?: (...args: unknown[]) => Promise<unknown> };
|
||||
const maybePGLite = engine as unknown as {
|
||||
db?: { query: (sql: string, params?: unknown[]) => Promise<{ rows: unknown[] }> };
|
||||
};
|
||||
|
||||
let rows: Array<{ id?: string; holder_pid?: number; holder_host?: string; acquired_at?: Date | string; ttl_expires_at?: Date | string; last_refreshed_at?: Date | string | null }>;
|
||||
|
||||
if (engine.kind === 'postgres' && maybePG.sql) {
|
||||
const sql = maybePG.sql as any;
|
||||
rows = await sql`
|
||||
SELECT id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at
|
||||
FROM gbrain_cycle_locks
|
||||
WHERE ttl_expires_at < NOW()
|
||||
ORDER BY acquired_at
|
||||
`;
|
||||
} else if (engine.kind === 'pglite' && maybePGLite.db) {
|
||||
const result = await maybePGLite.db.query(
|
||||
`SELECT id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at
|
||||
FROM gbrain_cycle_locks
|
||||
WHERE ttl_expires_at < NOW()
|
||||
ORDER BY acquired_at`,
|
||||
);
|
||||
rows = result.rows as typeof rows;
|
||||
} else {
|
||||
throw new Error(`Unknown engine kind for listStaleLocks: ${engine.kind}`);
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
return rows
|
||||
.filter(r => r.holder_pid !== undefined && r.acquired_at && r.ttl_expires_at)
|
||||
.map(r => {
|
||||
const acquired = r.acquired_at instanceof Date ? r.acquired_at : new Date(r.acquired_at!);
|
||||
const ttl = r.ttl_expires_at instanceof Date ? r.ttl_expires_at : new Date(r.ttl_expires_at!);
|
||||
// v0.41.13.0: last_refreshed_at may be NULL on pre-v98 brains.
|
||||
const lastRefreshed = r.last_refreshed_at == null
|
||||
? null
|
||||
: (r.last_refreshed_at instanceof Date ? r.last_refreshed_at : new Date(r.last_refreshed_at));
|
||||
return {
|
||||
id: String(r.id ?? ''),
|
||||
holder_pid: Number(r.holder_pid),
|
||||
holder_host: String(r.holder_host ?? ''),
|
||||
acquired_at: acquired,
|
||||
ttl_expires_at: ttl,
|
||||
age_ms: now - acquired.getTime(),
|
||||
ttl_expired: true,
|
||||
last_refreshed_at: lastRefreshed,
|
||||
ms_since_last_refresh: lastRefreshed ? now - lastRefreshed.getTime() : null,
|
||||
};
|
||||
});
|
||||
return selectLockRows(engine, { staleOnly: true });
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -555,6 +539,110 @@ export async function deleteLockRowIfStale(
|
||||
throw new Error(`Unknown engine kind for deleteLockRowIfStale: ${engine.kind}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* #1972 — snapshot-matched verify-and-delete for the background reaper.
|
||||
*
|
||||
* Unlike `deleteLockRow` (id + holder_pid only), this ALSO pins the observed
|
||||
* `acquired_at`, closing the TOCTOU window the reaper opens by reading rows
|
||||
* before deleting them: between the SELECT and the DELETE, the dead holder's
|
||||
* row could be replaced by a NEW holder that reused the same numeric PID
|
||||
* (PID-space wraps). That new row carries a newer `acquired_at`, so the match
|
||||
* fails and the DELETE is a safe no-op.
|
||||
*
|
||||
* Why `date_trunc('milliseconds', acquired_at) = $3` and not bare equality:
|
||||
* postgres.js parses timestamptz into a JS Date (millisecond precision), so the
|
||||
* snapshot we hold has already lost the microseconds that `acquired_at DEFAULT
|
||||
* NOW()` writes. A bare `acquired_at = $3` would therefore NEVER match in
|
||||
* production (microsecond stored value ≠ ms-truncated param) — the reaper would
|
||||
* silently delete nothing. Truncating both sides to ms makes the comparison
|
||||
* round-trip-safe on Postgres + PGLite, while a real takeover (seconds later)
|
||||
* still differs by far more than a millisecond.
|
||||
*/
|
||||
export async function deleteLockRowExact(
|
||||
engine: BrainEngine,
|
||||
lockId: string,
|
||||
holderPid: number,
|
||||
acquiredAt: Date,
|
||||
): Promise<{ deleted: boolean }> {
|
||||
const maybePG = engine as unknown as { sql?: (...args: unknown[]) => Promise<unknown> };
|
||||
const maybePGLite = engine as unknown as {
|
||||
db?: { query: (sql: string, params?: unknown[]) => Promise<{ rows: unknown[] }> };
|
||||
};
|
||||
|
||||
if (engine.kind === 'postgres' && maybePG.sql) {
|
||||
const sql = maybePG.sql as any;
|
||||
const rows: Array<{ id: string }> = await sql`
|
||||
DELETE FROM gbrain_cycle_locks
|
||||
WHERE id = ${lockId}
|
||||
AND holder_pid = ${holderPid}
|
||||
AND date_trunc('milliseconds', acquired_at) = ${acquiredAt}
|
||||
RETURNING id
|
||||
`;
|
||||
return { deleted: rows.length > 0 };
|
||||
}
|
||||
if (engine.kind === 'pglite' && maybePGLite.db) {
|
||||
const { rows } = await maybePGLite.db.query(
|
||||
`DELETE FROM gbrain_cycle_locks
|
||||
WHERE id = $1
|
||||
AND holder_pid = $2
|
||||
AND date_trunc('milliseconds', acquired_at) = $3
|
||||
RETURNING id`,
|
||||
[lockId, holderPid, acquiredAt],
|
||||
);
|
||||
return { deleted: rows.length > 0 };
|
||||
}
|
||||
throw new Error(`Unknown engine kind for deleteLockRowExact: ${engine.kind}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* #1972 — host-scoped reaper for dead-holder locks. Closes the gap where a sync
|
||||
* (or cycle) that crashed (OOM, recycle, SIGKILL) strands its lock row until
|
||||
* something contends for it — a low-traffic source could look "syncing" for
|
||||
* hours. `tryAcquireDbLock` only reclaims on contention; this is the background
|
||||
* sweep. Intended to run at cycle start (and under `gbrain doctor --fix` for
|
||||
* no-autopilot brains).
|
||||
*
|
||||
* Scoped to the `gbrain-sync:*` and `gbrain-cycle`/`gbrain-cycle:*` namespaces
|
||||
* ONLY. `gbrain_cycle_locks` is shared by enrich, the minion supervisor,
|
||||
* reindex, schema-pack, and elections (`tryWithDbElection`) — a blanket sweep
|
||||
* would change their TTL-failover timing. Those keep their existing
|
||||
* on-contention + TTL behavior.
|
||||
*
|
||||
* reapDeadHolderLocks (host-scoped, sync/cycle namespaces only)
|
||||
* selectLockRows → for each row:
|
||||
* ├─ id not in sync/cycle namespace ───────→ KEEP (blast-radius scope)
|
||||
* ├─ holder_host != thisHost ──────────────→ KEEP (cross_host; can't probe)
|
||||
* ├─ kill(pid,0) == alive / EPERM ─────────→ KEEP (live or not-ours)
|
||||
* ├─ ESRCH && age < 60s grace ─────────────→ KEEP (PID-reuse defense)
|
||||
* └─ ESRCH && age ≥ 60s grace ─────────────→ deleteLockRowExact(id, pid, acquired_at)
|
||||
* (snapshot-matched: reused-PID
|
||||
* fresh row has newer acquired_at → no-op)
|
||||
*/
|
||||
function isReapableNamespace(lockId: string): boolean {
|
||||
return (
|
||||
lockId === 'gbrain-cycle' ||
|
||||
lockId.startsWith('gbrain-cycle:') ||
|
||||
lockId.startsWith('gbrain-sync:')
|
||||
);
|
||||
}
|
||||
|
||||
export async function reapDeadHolderLocks(
|
||||
engine: BrainEngine,
|
||||
opts: HolderLivenessOpts = {},
|
||||
): Promise<{ reaped: number; reapedIds: string[] }> {
|
||||
const rows = await selectLockRows(engine);
|
||||
const reapedIds: string[] = [];
|
||||
for (const s of rows) {
|
||||
if (!isReapableNamespace(s.id)) continue;
|
||||
// isHolderDeadLocally combines same-host + ESRCH + the 60s reuse grace,
|
||||
// so pure-PID-liveness (which PID-space wrap defeats) is never used alone.
|
||||
if (!isHolderDeadLocally(s.holder_pid, s.holder_host, s.age_ms, opts)) continue;
|
||||
const { deleted } = await deleteLockRowExact(engine, s.id, s.holder_pid, s.acquired_at);
|
||||
if (deleted) reapedIds.push(s.id);
|
||||
}
|
||||
return { reaped: reapedIds.length, reapedIds };
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.40 (Federated Sync v2): per-source sync lock helper.
|
||||
*
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
/**
|
||||
* #1972 — host-scoped reaper for dead-holder sync/cycle locks.
|
||||
*
|
||||
* Covers:
|
||||
* - reapDeadHolderLocks: namespace scope (sync/cycle only, NOT election/etc),
|
||||
* same-host dead-PID reaped regardless of TTL, live/cross-host/within-grace
|
||||
* kept. Uses the injectable process.kill seam so it's deterministic.
|
||||
* - deleteLockRowExact: snapshot-matched delete (the TOCTOU defense) — a
|
||||
* non-matching acquired_at is a no-op, the matching one deletes.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
||||
import { hostname } from 'os';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import {
|
||||
reapDeadHolderLocks,
|
||||
deleteLockRowExact,
|
||||
inspectLock,
|
||||
HOLDER_TAKEOVER_GRACE_MS,
|
||||
type HolderLivenessOpts,
|
||||
} from '../src/core/db-lock.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({ database_url: '' });
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await engine.executeRaw(`DELETE FROM gbrain_cycle_locks`);
|
||||
});
|
||||
|
||||
const LOCAL = hostname();
|
||||
const OLD_S = Math.ceil(HOLDER_TAKEOVER_GRACE_MS / 1000) + 60; // comfortably past grace
|
||||
const YOUNG_S = 5;
|
||||
|
||||
/** Insert a lock row with a given age + namespace. ttlFuture controls whether
|
||||
* the row is TTL-expired (to prove the reaper reaps dead holders regardless). */
|
||||
async function seedLock(
|
||||
id: string,
|
||||
holderPid: number,
|
||||
holderHost: string,
|
||||
ageSeconds: number,
|
||||
ttlFuture = true,
|
||||
) {
|
||||
const ttl = ttlFuture ? `NOW() + INTERVAL '10 minutes'` : `NOW() - INTERVAL '1 minute'`;
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO gbrain_cycle_locks (id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at)
|
||||
VALUES ($1, $2, $3, NOW() - ($4 || ' seconds')::interval, ${ttl}, NOW() - ($4 || ' seconds')::interval)`,
|
||||
[id, holderPid, holderHost, String(ageSeconds)],
|
||||
);
|
||||
}
|
||||
|
||||
/** process.kill seam: every pid is dead (ESRCH) except those in `live`. */
|
||||
function killSeam(live: Set<number>): HolderLivenessOpts {
|
||||
return {
|
||||
localHost: LOCAL,
|
||||
processKill: (pid: number) => {
|
||||
if (live.has(pid)) return; // alive
|
||||
const e = new Error('no such process') as NodeJS.ErrnoException;
|
||||
e.code = 'ESRCH';
|
||||
throw e;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function lockIds(): Promise<string[]> {
|
||||
const rows = await engine.executeRaw<{ id: string }>(`SELECT id FROM gbrain_cycle_locks ORDER BY id`);
|
||||
return rows.map(r => r.id);
|
||||
}
|
||||
|
||||
describe('reapDeadHolderLocks', () => {
|
||||
test('reaps same-host dead-PID sync + cycle locks (regardless of TTL)', async () => {
|
||||
await seedLock('gbrain-sync:src-a', 900001, LOCAL, OLD_S, /*ttlFuture*/ true); // dead, TTL NOT expired
|
||||
await seedLock('gbrain-cycle', 900002, LOCAL, OLD_S, false); // dead, TTL expired
|
||||
await seedLock('gbrain-cycle:src-b', 900003, LOCAL, OLD_S, true); // dead, TTL NOT expired
|
||||
|
||||
const { reaped, reapedIds } = await reapDeadHolderLocks(engine, killSeam(new Set()));
|
||||
expect(reaped).toBe(3);
|
||||
expect(reapedIds.sort()).toEqual(['gbrain-cycle', 'gbrain-cycle:src-b', 'gbrain-sync:src-a']);
|
||||
expect(await lockIds()).toEqual([]);
|
||||
});
|
||||
|
||||
test('keeps a live same-host holder', async () => {
|
||||
await seedLock('gbrain-sync:src-live', process.pid, LOCAL, OLD_S);
|
||||
const { reaped } = await reapDeadHolderLocks(engine, killSeam(new Set([process.pid])));
|
||||
expect(reaped).toBe(0);
|
||||
expect(await lockIds()).toEqual(['gbrain-sync:src-live']);
|
||||
});
|
||||
|
||||
test('keeps a dead holder still within the PID-reuse grace window', async () => {
|
||||
await seedLock('gbrain-sync:src-young', 900010, LOCAL, YOUNG_S);
|
||||
const { reaped } = await reapDeadHolderLocks(engine, killSeam(new Set()));
|
||||
expect(reaped).toBe(0);
|
||||
expect(await lockIds()).toEqual(['gbrain-sync:src-young']);
|
||||
});
|
||||
|
||||
test('keeps a cross-host holder (cannot probe a remote PID)', async () => {
|
||||
await seedLock('gbrain-sync:src-xhost', 900011, 'a-different-host', OLD_S);
|
||||
const { reaped } = await reapDeadHolderLocks(engine, killSeam(new Set()));
|
||||
expect(reaped).toBe(0);
|
||||
expect(await lockIds()).toEqual(['gbrain-sync:src-xhost']);
|
||||
});
|
||||
|
||||
test('NEVER reaps a non-sync/cycle namespace, even with a dead PID (blast radius)', async () => {
|
||||
await seedLock('gbrain-election:leader', 900020, LOCAL, OLD_S);
|
||||
await seedLock('some-other-lock', 900021, LOCAL, OLD_S);
|
||||
const { reaped } = await reapDeadHolderLocks(engine, killSeam(new Set()));
|
||||
expect(reaped).toBe(0);
|
||||
expect(await lockIds()).toEqual(['gbrain-election:leader', 'some-other-lock']);
|
||||
});
|
||||
|
||||
test('empty table → {reaped:0}', async () => {
|
||||
const r = await reapDeadHolderLocks(engine, killSeam(new Set()));
|
||||
expect(r).toEqual({ reaped: 0, reapedIds: [] });
|
||||
});
|
||||
|
||||
test('mixed set: reaps only the eligible rows', async () => {
|
||||
await seedLock('gbrain-sync:dead', 900030, LOCAL, OLD_S); // reap
|
||||
await seedLock('gbrain-sync:live', process.pid, LOCAL, OLD_S); // keep (live)
|
||||
await seedLock('gbrain-cycle:xhost', 900031, 'other', OLD_S); // keep (cross-host)
|
||||
await seedLock('gbrain-election:x', 900032, LOCAL, OLD_S); // keep (namespace)
|
||||
const { reaped, reapedIds } = await reapDeadHolderLocks(engine, killSeam(new Set([process.pid])));
|
||||
expect(reaped).toBe(1);
|
||||
expect(reapedIds).toEqual(['gbrain-sync:dead']);
|
||||
expect(await lockIds()).toEqual(['gbrain-cycle:xhost', 'gbrain-election:x', 'gbrain-sync:live']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('deleteLockRowExact (snapshot-matched, TOCTOU defense)', () => {
|
||||
test('no-op when acquired_at does not match; deletes when it does', async () => {
|
||||
await seedLock('gbrain-sync:exact', 900040, LOCAL, OLD_S);
|
||||
const snap = await inspectLock(engine, 'gbrain-sync:exact');
|
||||
expect(snap).not.toBeNull();
|
||||
|
||||
// Simulate a reused-PID takeover: same id + pid, but a different (newer)
|
||||
// acquired_at than the one the reaper snapshotted → must NOT delete.
|
||||
const wrong = new Date(snap!.acquired_at.getTime() + 3_600_000);
|
||||
const miss = await deleteLockRowExact(engine, 'gbrain-sync:exact', 900040, wrong);
|
||||
expect(miss.deleted).toBe(false);
|
||||
expect(await lockIds()).toEqual(['gbrain-sync:exact']);
|
||||
|
||||
// The matching snapshot deletes.
|
||||
const hit = await deleteLockRowExact(engine, 'gbrain-sync:exact', 900040, snap!.acquired_at);
|
||||
expect(hit.deleted).toBe(true);
|
||||
expect(await lockIds()).toEqual([]);
|
||||
});
|
||||
|
||||
test('no-op when holder_pid does not match', async () => {
|
||||
await seedLock('gbrain-sync:pidguard', 900050, LOCAL, OLD_S);
|
||||
const snap = await inspectLock(engine, 'gbrain-sync:pidguard');
|
||||
const res = await deleteLockRowExact(engine, 'gbrain-sync:pidguard', 111111, snap!.acquired_at);
|
||||
expect(res.deleted).toBe(false);
|
||||
expect(await lockIds()).toEqual(['gbrain-sync:pidguard']);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user