fix(jobs): reap stale dead-holder cycle/sync locks (#1972)

A crashed sync (OOM, recycle, SIGKILL) stranded its gbrain_cycle_locks row
until something contended for it — reclaim was on-contention only. Add a
host-scoped background reaper: reapDeadHolderLocks deletes locks whose holder
PID is provably dead on this host, scoped to the gbrain-sync:*/gbrain-cycle*
namespaces only (never elections/supervisor/reindex), with a snapshot-matched
delete (date_trunc on acquired_at) that is TOCTOU-safe against PID reuse.
Reuses isHolderDeadLocally (same-host + ESRCH + 60s grace). doctor --fix now
auto-reaps for no-autopilot brains. DRY: selectLockRows + shared mapper now
back inspectLock + listStaleLocks (killed the triplication).
This commit is contained in:
Garry Tan
2026-06-09 08:35:45 -07:00
parent 099d9a8f55
commit 2f9e2136da
3 changed files with 380 additions and 99 deletions
+36 -5
View File
@@ -2346,12 +2346,39 @@ export function checkAutopilotLockScope(): Check {
* but the main work is blocked. Requires explicit heartbeat probe;
* speculation until production data shows the case.
*/
export async function checkStaleLocks(engine: BrainEngine): Promise<Check> {
export async function checkStaleLocks(
engine: BrainEngine,
opts: { fix?: boolean; dryRun?: boolean } = {},
): Promise<Check> {
try {
const { listStaleLocks } = await import('../core/db-lock.ts');
const { listStaleLocks, reapDeadHolderLocks } = await import('../core/db-lock.ts');
// #1972: under `gbrain doctor --fix`, reap dead-holder sync/cycle locks
// using the SAME namespace-scoped, host-scoped, snapshot-matched reaper the
// cycle runs at start. This is the self-heal path for no-autopilot brains: a
// brain that never runs `gbrain dream` never hits the cycle-start sweep, so
// doctor --fix is how its crashed-sync locks get cleared. DB-only, so it's
// orthogonal to (and unaffected by) the skills-dir --fix safety gate above.
// Best-effort: a reap failure falls through to the warn path below.
let reapedIds: string[] = [];
if (opts.fix && !opts.dryRun) {
try {
reapedIds = (await reapDeadHolderLocks(engine)).reapedIds;
} catch { /* fall through; listStaleLocks still surfaces remaining locks */ }
}
const reapedNote = reapedIds.length > 0
? `Reaped ${reapedIds.length} dead-holder lock(s): ${reapedIds.join(', ')}.`
: null;
const stale = await listStaleLocks(engine);
if (stale.length === 0) {
return { name: 'stale_locks', status: 'ok', message: 'No stale locks (no rows with ttl_expires_at < NOW())' };
return {
name: 'stale_locks',
status: 'ok',
message: reapedNote
? `${reapedNote} No stale locks remain.`
: 'No stale locks (no rows with ttl_expires_at < NOW())',
};
}
const lines = stale.slice(0, 10).map(s => {
const ageH = Math.floor(s.age_ms / 3600_000);
@@ -2360,11 +2387,15 @@ export async function checkStaleLocks(engine: BrainEngine): Promise<Check> {
return ` ${s.id} (pid ${s.holder_pid} on ${s.holder_host}, age ${ageH}h) → ${breakHint}`;
});
const tail = stale.length > 10 ? ` ... and ${stale.length - 10} more.` : null;
const header = opts.fix
? `${stale.length} stale lock(s) remain that could not be auto-reaped (live holder, cross-host, or within the PID-reuse grace):`
: `${stale.length} stale lock(s) detected (ttl_expires_at < NOW()):`;
return {
name: 'stale_locks',
status: 'warn',
message: [
`${stale.length} stale lock(s) detected (ttl_expires_at < NOW()):`,
reapedNote,
header,
...lines,
tail,
].filter(Boolean).join('\n'),
@@ -6951,7 +6982,7 @@ export async function buildChecks(
checks.push(checkAutopilotLockScope());
// v0.41.6.0 D3 — stale_locks (gbrain_cycle_locks rows with ttl_expires_at < NOW())
progress.heartbeat('stale_locks');
checks.push(await checkStaleLocks(engine));
checks.push(await checkStaleLocks(engine, { fix: doFix, dryRun }));
// v0.38 — cycle_phase_scope (informational; no DB cost)
progress.heartbeat('cycle_phase_scope');
checks.push(checkCyclePhaseScope());
+182 -94
View File
@@ -307,58 +307,37 @@ export interface LockSnapshot {
ms_since_last_refresh: number | null;
}
export async function inspectLock(engine: BrainEngine, lockId: string): Promise<LockSnapshot | null> {
const maybePG = engine as unknown as { sql?: (...args: unknown[]) => Promise<unknown> };
const maybePGLite = engine as unknown as {
db?: { query: (sql: string, params?: unknown[]) => Promise<{ rows: unknown[] }> };
};
/**
* Raw row shape returned by every `gbrain_cycle_locks` SELECT. Lives here so
* `selectLockRows` (the single canonical reader) and its mapper share one type.
*/
interface RawLockRow {
id?: string;
holder_pid?: number;
holder_host?: string;
acquired_at?: Date | string;
ttl_expires_at?: Date | string;
last_refreshed_at?: Date | string | null;
}
let row: {
id?: string;
holder_pid?: number;
holder_host?: string;
acquired_at?: Date | string;
ttl_expires_at?: Date | string;
last_refreshed_at?: Date | string | null;
} | undefined;
if (engine.kind === 'postgres' && maybePG.sql) {
const sql = maybePG.sql as any;
const rows = await sql`
SELECT id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at
FROM gbrain_cycle_locks
WHERE id = ${lockId}
`;
row = rows[0];
} else if (engine.kind === 'pglite' && maybePGLite.db) {
const { rows } = await maybePGLite.db.query(
`SELECT id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at
FROM gbrain_cycle_locks
WHERE id = $1`,
[lockId],
);
row = rows[0] as typeof row;
} else {
throw new Error(`Unknown engine kind for inspectLock: ${engine.kind}`);
}
if (!row || row.holder_pid === undefined || !row.acquired_at || !row.ttl_expires_at) return null;
/** Canonical column list for every lock SELECT — keep in lockstep with `RawLockRow`. */
const LOCK_SELECT_COLS = 'id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at';
/**
* Row → `LockSnapshot` mapper. Coerces postgres.js Date|string columns to Date
* and computes the derived fields (age_ms, ttl_expired, ms_since_last_refresh).
* Returns null for a structurally-incomplete row (missing pid / acquired_at /
* ttl). v0.41.13.0: last_refreshed_at may be NULL on pre-v98 brains.
*/
function rowToLockSnapshot(row: RawLockRow, now: number): LockSnapshot | null {
if (row.holder_pid === undefined || !row.acquired_at || !row.ttl_expires_at) return null;
const acquired = row.acquired_at instanceof Date ? row.acquired_at : new Date(row.acquired_at);
const ttlExpires = row.ttl_expires_at instanceof Date ? row.ttl_expires_at : new Date(row.ttl_expires_at);
const now = Date.now();
// v0.41.13.0: last_refreshed_at may be NULL on pre-v98 brains that have
// the column but no acquire has happened since the migration ran. Render
// both `last_refreshed_at` and the computed delta as null so callers can
// distinguish "never observed a refresh" from "refresh fired N ms ago".
const lastRefreshed = row.last_refreshed_at == null
? null
: (row.last_refreshed_at instanceof Date
? row.last_refreshed_at
: new Date(row.last_refreshed_at));
: (row.last_refreshed_at instanceof Date ? row.last_refreshed_at : new Date(row.last_refreshed_at));
return {
id: lockId,
id: String(row.id ?? ''),
holder_pid: Number(row.holder_pid),
holder_host: String(row.holder_host ?? ''),
acquired_at: acquired,
@@ -370,6 +349,59 @@ export async function inspectLock(engine: BrainEngine, lockId: string): Promise<
};
}
interface SelectLockRowsOpts {
/** Return only the row for this exact lock id (the `inspectLock` case). */
lockId?: string;
/** Return only rows whose ttl_expires_at < NOW() (the `listStaleLocks` case). */
staleOnly?: boolean;
}
/**
* The single canonical reader for `gbrain_cycle_locks`. Engine-branches once
* (postgres / pglite) so `inspectLock`, `listStaleLocks`, and the reaper don't
* each re-roll the SELECT + Date-coercion (previously triplicated). Returns
* fully-mapped `LockSnapshot[]`; callers filter in JS (the table holds 0-2 rows
* in practice, so a no-WHERE full read is cheap and keeps the SQL trivial).
*/
async function selectLockRows(engine: BrainEngine, opts: SelectLockRowsOpts = {}): Promise<LockSnapshot[]> {
const maybePG = engine as unknown as { sql?: (...args: unknown[]) => Promise<unknown> };
const maybePGLite = engine as unknown as {
db?: { query: (sql: string, params?: unknown[]) => Promise<{ rows: unknown[] }> };
};
let rows: RawLockRow[];
if (engine.kind === 'postgres' && maybePG.sql) {
const sql = maybePG.sql as any;
if (opts.lockId !== undefined) {
rows = await sql`SELECT id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at FROM gbrain_cycle_locks WHERE id = ${opts.lockId}`;
} else if (opts.staleOnly) {
rows = await sql`SELECT id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at FROM gbrain_cycle_locks WHERE ttl_expires_at < NOW() ORDER BY acquired_at`;
} else {
rows = await sql`SELECT id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at FROM gbrain_cycle_locks ORDER BY acquired_at`;
}
} else if (engine.kind === 'pglite' && maybePGLite.db) {
if (opts.lockId !== undefined) {
rows = (await maybePGLite.db.query(`SELECT ${LOCK_SELECT_COLS} FROM gbrain_cycle_locks WHERE id = $1`, [opts.lockId])).rows as RawLockRow[];
} else if (opts.staleOnly) {
rows = (await maybePGLite.db.query(`SELECT ${LOCK_SELECT_COLS} FROM gbrain_cycle_locks WHERE ttl_expires_at < NOW() ORDER BY acquired_at`)).rows as RawLockRow[];
} else {
rows = (await maybePGLite.db.query(`SELECT ${LOCK_SELECT_COLS} FROM gbrain_cycle_locks ORDER BY acquired_at`)).rows as RawLockRow[];
}
} else {
throw new Error(`Unknown engine kind for selectLockRows: ${engine.kind}`);
}
const now = Date.now();
return rows
.map((r) => rowToLockSnapshot(r, now))
.filter((s): s is LockSnapshot => s !== null);
}
export async function inspectLock(engine: BrainEngine, lockId: string): Promise<LockSnapshot | null> {
const rows = await selectLockRows(engine, { lockId });
return rows[0] ?? null;
}
/**
* v0.41.6.0 D3: list every lock whose TTL has expired. Used by gbrain
* doctor's `stale_locks` check. The query reuses the same canonical
@@ -377,55 +409,7 @@ export async function inspectLock(engine: BrainEngine, lockId: string): Promise<
* UPDATE-on-conflict already trusts — no parallel heuristic.
*/
export async function listStaleLocks(engine: BrainEngine): Promise<LockSnapshot[]> {
const maybePG = engine as unknown as { sql?: (...args: unknown[]) => Promise<unknown> };
const maybePGLite = engine as unknown as {
db?: { query: (sql: string, params?: unknown[]) => Promise<{ rows: unknown[] }> };
};
let rows: Array<{ id?: string; holder_pid?: number; holder_host?: string; acquired_at?: Date | string; ttl_expires_at?: Date | string; last_refreshed_at?: Date | string | null }>;
if (engine.kind === 'postgres' && maybePG.sql) {
const sql = maybePG.sql as any;
rows = await sql`
SELECT id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at
FROM gbrain_cycle_locks
WHERE ttl_expires_at < NOW()
ORDER BY acquired_at
`;
} else if (engine.kind === 'pglite' && maybePGLite.db) {
const result = await maybePGLite.db.query(
`SELECT id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at
FROM gbrain_cycle_locks
WHERE ttl_expires_at < NOW()
ORDER BY acquired_at`,
);
rows = result.rows as typeof rows;
} else {
throw new Error(`Unknown engine kind for listStaleLocks: ${engine.kind}`);
}
const now = Date.now();
return rows
.filter(r => r.holder_pid !== undefined && r.acquired_at && r.ttl_expires_at)
.map(r => {
const acquired = r.acquired_at instanceof Date ? r.acquired_at : new Date(r.acquired_at!);
const ttl = r.ttl_expires_at instanceof Date ? r.ttl_expires_at : new Date(r.ttl_expires_at!);
// v0.41.13.0: last_refreshed_at may be NULL on pre-v98 brains.
const lastRefreshed = r.last_refreshed_at == null
? null
: (r.last_refreshed_at instanceof Date ? r.last_refreshed_at : new Date(r.last_refreshed_at));
return {
id: String(r.id ?? ''),
holder_pid: Number(r.holder_pid),
holder_host: String(r.holder_host ?? ''),
acquired_at: acquired,
ttl_expires_at: ttl,
age_ms: now - acquired.getTime(),
ttl_expired: true,
last_refreshed_at: lastRefreshed,
ms_since_last_refresh: lastRefreshed ? now - lastRefreshed.getTime() : null,
};
});
return selectLockRows(engine, { staleOnly: true });
}
/**
@@ -555,6 +539,110 @@ export async function deleteLockRowIfStale(
throw new Error(`Unknown engine kind for deleteLockRowIfStale: ${engine.kind}`);
}
/**
* #1972 — snapshot-matched verify-and-delete for the background reaper.
*
* Unlike `deleteLockRow` (id + holder_pid only), this ALSO pins the observed
* `acquired_at`, closing the TOCTOU window the reaper opens by reading rows
* before deleting them: between the SELECT and the DELETE, the dead holder's
* row could be replaced by a NEW holder that reused the same numeric PID
* (PID-space wraps). That new row carries a newer `acquired_at`, so the match
* fails and the DELETE is a safe no-op.
*
* Why `date_trunc('milliseconds', acquired_at) = $3` and not bare equality:
* postgres.js parses timestamptz into a JS Date (millisecond precision), so the
* snapshot we hold has already lost the microseconds that `acquired_at DEFAULT
* NOW()` writes. A bare `acquired_at = $3` would therefore NEVER match in
* production (microsecond stored value ≠ ms-truncated param) — the reaper would
* silently delete nothing. Truncating both sides to ms makes the comparison
* round-trip-safe on Postgres + PGLite, while a real takeover (seconds later)
* still differs by far more than a millisecond.
*/
export async function deleteLockRowExact(
engine: BrainEngine,
lockId: string,
holderPid: number,
acquiredAt: Date,
): Promise<{ deleted: boolean }> {
const maybePG = engine as unknown as { sql?: (...args: unknown[]) => Promise<unknown> };
const maybePGLite = engine as unknown as {
db?: { query: (sql: string, params?: unknown[]) => Promise<{ rows: unknown[] }> };
};
if (engine.kind === 'postgres' && maybePG.sql) {
const sql = maybePG.sql as any;
const rows: Array<{ id: string }> = await sql`
DELETE FROM gbrain_cycle_locks
WHERE id = ${lockId}
AND holder_pid = ${holderPid}
AND date_trunc('milliseconds', acquired_at) = ${acquiredAt}
RETURNING id
`;
return { deleted: rows.length > 0 };
}
if (engine.kind === 'pglite' && maybePGLite.db) {
const { rows } = await maybePGLite.db.query(
`DELETE FROM gbrain_cycle_locks
WHERE id = $1
AND holder_pid = $2
AND date_trunc('milliseconds', acquired_at) = $3
RETURNING id`,
[lockId, holderPid, acquiredAt],
);
return { deleted: rows.length > 0 };
}
throw new Error(`Unknown engine kind for deleteLockRowExact: ${engine.kind}`);
}
/**
* #1972 — host-scoped reaper for dead-holder locks. Closes the gap where a sync
* (or cycle) that crashed (OOM, recycle, SIGKILL) strands its lock row until
* something contends for it — a low-traffic source could look "syncing" for
* hours. `tryAcquireDbLock` only reclaims on contention; this is the background
* sweep. Intended to run at cycle start (and under `gbrain doctor --fix` for
* no-autopilot brains).
*
* Scoped to the `gbrain-sync:*` and `gbrain-cycle`/`gbrain-cycle:*` namespaces
* ONLY. `gbrain_cycle_locks` is shared by enrich, the minion supervisor,
* reindex, schema-pack, and elections (`tryWithDbElection`) — a blanket sweep
* would change their TTL-failover timing. Those keep their existing
* on-contention + TTL behavior.
*
* reapDeadHolderLocks (host-scoped, sync/cycle namespaces only)
* selectLockRows → for each row:
* ├─ id not in sync/cycle namespace ───────→ KEEP (blast-radius scope)
* ├─ holder_host != thisHost ──────────────→ KEEP (cross_host; can't probe)
* ├─ kill(pid,0) == alive / EPERM ─────────→ KEEP (live or not-ours)
* ├─ ESRCH && age < 60s grace ─────────────→ KEEP (PID-reuse defense)
* └─ ESRCH && age ≥ 60s grace ─────────────→ deleteLockRowExact(id, pid, acquired_at)
* (snapshot-matched: reused-PID
* fresh row has newer acquired_at → no-op)
*/
function isReapableNamespace(lockId: string): boolean {
return (
lockId === 'gbrain-cycle' ||
lockId.startsWith('gbrain-cycle:') ||
lockId.startsWith('gbrain-sync:')
);
}
export async function reapDeadHolderLocks(
engine: BrainEngine,
opts: HolderLivenessOpts = {},
): Promise<{ reaped: number; reapedIds: string[] }> {
const rows = await selectLockRows(engine);
const reapedIds: string[] = [];
for (const s of rows) {
if (!isReapableNamespace(s.id)) continue;
// isHolderDeadLocally combines same-host + ESRCH + the 60s reuse grace,
// so pure-PID-liveness (which PID-space wrap defeats) is never used alone.
if (!isHolderDeadLocally(s.holder_pid, s.holder_host, s.age_ms, opts)) continue;
const { deleted } = await deleteLockRowExact(engine, s.id, s.holder_pid, s.acquired_at);
if (deleted) reapedIds.push(s.id);
}
return { reaped: reapedIds.length, reapedIds };
}
/**
* v0.40 (Federated Sync v2): per-source sync lock helper.
*
+162
View File
@@ -0,0 +1,162 @@
/**
* #1972 — host-scoped reaper for dead-holder sync/cycle locks.
*
* Covers:
* - reapDeadHolderLocks: namespace scope (sync/cycle only, NOT election/etc),
* same-host dead-PID reaped regardless of TTL, live/cross-host/within-grace
* kept. Uses the injectable process.kill seam so it's deterministic.
* - deleteLockRowExact: snapshot-matched delete (the TOCTOU defense) — a
* non-matching acquired_at is a no-op, the matching one deletes.
*/
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
import { hostname } from 'os';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import {
reapDeadHolderLocks,
deleteLockRowExact,
inspectLock,
HOLDER_TAKEOVER_GRACE_MS,
type HolderLivenessOpts,
} from '../src/core/db-lock.ts';
let engine: PGLiteEngine;
beforeAll(async () => {
engine = new PGLiteEngine();
await engine.connect({ database_url: '' });
await engine.initSchema();
});
afterAll(async () => {
await engine.disconnect();
});
beforeEach(async () => {
await engine.executeRaw(`DELETE FROM gbrain_cycle_locks`);
});
const LOCAL = hostname();
const OLD_S = Math.ceil(HOLDER_TAKEOVER_GRACE_MS / 1000) + 60; // comfortably past grace
const YOUNG_S = 5;
/** Insert a lock row with a given age + namespace. ttlFuture controls whether
* the row is TTL-expired (to prove the reaper reaps dead holders regardless). */
async function seedLock(
id: string,
holderPid: number,
holderHost: string,
ageSeconds: number,
ttlFuture = true,
) {
const ttl = ttlFuture ? `NOW() + INTERVAL '10 minutes'` : `NOW() - INTERVAL '1 minute'`;
await engine.executeRaw(
`INSERT INTO gbrain_cycle_locks (id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at)
VALUES ($1, $2, $3, NOW() - ($4 || ' seconds')::interval, ${ttl}, NOW() - ($4 || ' seconds')::interval)`,
[id, holderPid, holderHost, String(ageSeconds)],
);
}
/** process.kill seam: every pid is dead (ESRCH) except those in `live`. */
function killSeam(live: Set<number>): HolderLivenessOpts {
return {
localHost: LOCAL,
processKill: (pid: number) => {
if (live.has(pid)) return; // alive
const e = new Error('no such process') as NodeJS.ErrnoException;
e.code = 'ESRCH';
throw e;
},
};
}
async function lockIds(): Promise<string[]> {
const rows = await engine.executeRaw<{ id: string }>(`SELECT id FROM gbrain_cycle_locks ORDER BY id`);
return rows.map(r => r.id);
}
describe('reapDeadHolderLocks', () => {
test('reaps same-host dead-PID sync + cycle locks (regardless of TTL)', async () => {
await seedLock('gbrain-sync:src-a', 900001, LOCAL, OLD_S, /*ttlFuture*/ true); // dead, TTL NOT expired
await seedLock('gbrain-cycle', 900002, LOCAL, OLD_S, false); // dead, TTL expired
await seedLock('gbrain-cycle:src-b', 900003, LOCAL, OLD_S, true); // dead, TTL NOT expired
const { reaped, reapedIds } = await reapDeadHolderLocks(engine, killSeam(new Set()));
expect(reaped).toBe(3);
expect(reapedIds.sort()).toEqual(['gbrain-cycle', 'gbrain-cycle:src-b', 'gbrain-sync:src-a']);
expect(await lockIds()).toEqual([]);
});
test('keeps a live same-host holder', async () => {
await seedLock('gbrain-sync:src-live', process.pid, LOCAL, OLD_S);
const { reaped } = await reapDeadHolderLocks(engine, killSeam(new Set([process.pid])));
expect(reaped).toBe(0);
expect(await lockIds()).toEqual(['gbrain-sync:src-live']);
});
test('keeps a dead holder still within the PID-reuse grace window', async () => {
await seedLock('gbrain-sync:src-young', 900010, LOCAL, YOUNG_S);
const { reaped } = await reapDeadHolderLocks(engine, killSeam(new Set()));
expect(reaped).toBe(0);
expect(await lockIds()).toEqual(['gbrain-sync:src-young']);
});
test('keeps a cross-host holder (cannot probe a remote PID)', async () => {
await seedLock('gbrain-sync:src-xhost', 900011, 'a-different-host', OLD_S);
const { reaped } = await reapDeadHolderLocks(engine, killSeam(new Set()));
expect(reaped).toBe(0);
expect(await lockIds()).toEqual(['gbrain-sync:src-xhost']);
});
test('NEVER reaps a non-sync/cycle namespace, even with a dead PID (blast radius)', async () => {
await seedLock('gbrain-election:leader', 900020, LOCAL, OLD_S);
await seedLock('some-other-lock', 900021, LOCAL, OLD_S);
const { reaped } = await reapDeadHolderLocks(engine, killSeam(new Set()));
expect(reaped).toBe(0);
expect(await lockIds()).toEqual(['gbrain-election:leader', 'some-other-lock']);
});
test('empty table → {reaped:0}', async () => {
const r = await reapDeadHolderLocks(engine, killSeam(new Set()));
expect(r).toEqual({ reaped: 0, reapedIds: [] });
});
test('mixed set: reaps only the eligible rows', async () => {
await seedLock('gbrain-sync:dead', 900030, LOCAL, OLD_S); // reap
await seedLock('gbrain-sync:live', process.pid, LOCAL, OLD_S); // keep (live)
await seedLock('gbrain-cycle:xhost', 900031, 'other', OLD_S); // keep (cross-host)
await seedLock('gbrain-election:x', 900032, LOCAL, OLD_S); // keep (namespace)
const { reaped, reapedIds } = await reapDeadHolderLocks(engine, killSeam(new Set([process.pid])));
expect(reaped).toBe(1);
expect(reapedIds).toEqual(['gbrain-sync:dead']);
expect(await lockIds()).toEqual(['gbrain-cycle:xhost', 'gbrain-election:x', 'gbrain-sync:live']);
});
});
describe('deleteLockRowExact (snapshot-matched, TOCTOU defense)', () => {
test('no-op when acquired_at does not match; deletes when it does', async () => {
await seedLock('gbrain-sync:exact', 900040, LOCAL, OLD_S);
const snap = await inspectLock(engine, 'gbrain-sync:exact');
expect(snap).not.toBeNull();
// Simulate a reused-PID takeover: same id + pid, but a different (newer)
// acquired_at than the one the reaper snapshotted → must NOT delete.
const wrong = new Date(snap!.acquired_at.getTime() + 3_600_000);
const miss = await deleteLockRowExact(engine, 'gbrain-sync:exact', 900040, wrong);
expect(miss.deleted).toBe(false);
expect(await lockIds()).toEqual(['gbrain-sync:exact']);
// The matching snapshot deletes.
const hit = await deleteLockRowExact(engine, 'gbrain-sync:exact', 900040, snap!.acquired_at);
expect(hit.deleted).toBe(true);
expect(await lockIds()).toEqual([]);
});
test('no-op when holder_pid does not match', async () => {
await seedLock('gbrain-sync:pidguard', 900050, LOCAL, OLD_S);
const snap = await inspectLock(engine, 'gbrain-sync:pidguard');
const res = await deleteLockRowExact(engine, 'gbrain-sync:pidguard', 111111, snap!.acquired_at);
expect(res.deleted).toBe(false);
expect(await lockIds()).toEqual(['gbrain-sync:pidguard']);
});
});