diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index b59e6fc8b..08651157e 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -2346,12 +2346,39 @@ export function checkAutopilotLockScope(): Check { * but the main work is blocked. Requires explicit heartbeat probe; * speculation until production data shows the case. */ -export async function checkStaleLocks(engine: BrainEngine): Promise { +export async function checkStaleLocks( + engine: BrainEngine, + opts: { fix?: boolean; dryRun?: boolean } = {}, +): Promise { try { - const { listStaleLocks } = await import('../core/db-lock.ts'); + const { listStaleLocks, reapDeadHolderLocks } = await import('../core/db-lock.ts'); + + // #1972: under `gbrain doctor --fix`, reap dead-holder sync/cycle locks + // using the SAME namespace-scoped, host-scoped, snapshot-matched reaper the + // cycle runs at start. This is the self-heal path for no-autopilot brains: a + // brain that never runs `gbrain dream` never hits the cycle-start sweep, so + // doctor --fix is how its crashed-sync locks get cleared. DB-only, so it's + // orthogonal to (and unaffected by) the skills-dir --fix safety gate above. + // Best-effort: a reap failure falls through to the warn path below. + let reapedIds: string[] = []; + if (opts.fix && !opts.dryRun) { + try { + reapedIds = (await reapDeadHolderLocks(engine)).reapedIds; + } catch { /* fall through; listStaleLocks still surfaces remaining locks */ } + } + const reapedNote = reapedIds.length > 0 + ? `Reaped ${reapedIds.length} dead-holder lock(s): ${reapedIds.join(', ')}.` + : null; + const stale = await listStaleLocks(engine); if (stale.length === 0) { - return { name: 'stale_locks', status: 'ok', message: 'No stale locks (no rows with ttl_expires_at < NOW())' }; + return { + name: 'stale_locks', + status: 'ok', + message: reapedNote + ? `${reapedNote} No stale locks remain.` + : 'No stale locks (no rows with ttl_expires_at < NOW())', + }; } const lines = stale.slice(0, 10).map(s => { const ageH = Math.floor(s.age_ms / 3600_000); @@ -2360,11 +2387,15 @@ export async function checkStaleLocks(engine: BrainEngine): Promise { return ` ${s.id} (pid ${s.holder_pid} on ${s.holder_host}, age ${ageH}h) → ${breakHint}`; }); const tail = stale.length > 10 ? ` ... and ${stale.length - 10} more.` : null; + const header = opts.fix + ? `${stale.length} stale lock(s) remain that could not be auto-reaped (live holder, cross-host, or within the PID-reuse grace):` + : `${stale.length} stale lock(s) detected (ttl_expires_at < NOW()):`; return { name: 'stale_locks', status: 'warn', message: [ - `${stale.length} stale lock(s) detected (ttl_expires_at < NOW()):`, + reapedNote, + header, ...lines, tail, ].filter(Boolean).join('\n'), @@ -6951,7 +6982,7 @@ export async function buildChecks( checks.push(checkAutopilotLockScope()); // v0.41.6.0 D3 — stale_locks (gbrain_cycle_locks rows with ttl_expires_at < NOW()) progress.heartbeat('stale_locks'); - checks.push(await checkStaleLocks(engine)); + checks.push(await checkStaleLocks(engine, { fix: doFix, dryRun })); // v0.38 — cycle_phase_scope (informational; no DB cost) progress.heartbeat('cycle_phase_scope'); checks.push(checkCyclePhaseScope()); diff --git a/src/core/db-lock.ts b/src/core/db-lock.ts index 4c01e4ce2..dd5de5dc7 100644 --- a/src/core/db-lock.ts +++ b/src/core/db-lock.ts @@ -307,58 +307,37 @@ export interface LockSnapshot { ms_since_last_refresh: number | null; } -export async function inspectLock(engine: BrainEngine, lockId: string): Promise { - const maybePG = engine as unknown as { sql?: (...args: unknown[]) => Promise }; - const maybePGLite = engine as unknown as { - db?: { query: (sql: string, params?: unknown[]) => Promise<{ rows: unknown[] }> }; - }; +/** + * Raw row shape returned by every `gbrain_cycle_locks` SELECT. Lives here so + * `selectLockRows` (the single canonical reader) and its mapper share one type. + */ +interface RawLockRow { + id?: string; + holder_pid?: number; + holder_host?: string; + acquired_at?: Date | string; + ttl_expires_at?: Date | string; + last_refreshed_at?: Date | string | null; +} - let row: { - id?: string; - holder_pid?: number; - holder_host?: string; - acquired_at?: Date | string; - ttl_expires_at?: Date | string; - last_refreshed_at?: Date | string | null; - } | undefined; - - if (engine.kind === 'postgres' && maybePG.sql) { - const sql = maybePG.sql as any; - const rows = await sql` - SELECT id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at - FROM gbrain_cycle_locks - WHERE id = ${lockId} - `; - row = rows[0]; - } else if (engine.kind === 'pglite' && maybePGLite.db) { - const { rows } = await maybePGLite.db.query( - `SELECT id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at - FROM gbrain_cycle_locks - WHERE id = $1`, - [lockId], - ); - row = rows[0] as typeof row; - } else { - throw new Error(`Unknown engine kind for inspectLock: ${engine.kind}`); - } - - if (!row || row.holder_pid === undefined || !row.acquired_at || !row.ttl_expires_at) return null; +/** Canonical column list for every lock SELECT — keep in lockstep with `RawLockRow`. */ +const LOCK_SELECT_COLS = 'id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at'; +/** + * Row → `LockSnapshot` mapper. Coerces postgres.js Date|string columns to Date + * and computes the derived fields (age_ms, ttl_expired, ms_since_last_refresh). + * Returns null for a structurally-incomplete row (missing pid / acquired_at / + * ttl). v0.41.13.0: last_refreshed_at may be NULL on pre-v98 brains. + */ +function rowToLockSnapshot(row: RawLockRow, now: number): LockSnapshot | null { + if (row.holder_pid === undefined || !row.acquired_at || !row.ttl_expires_at) return null; const acquired = row.acquired_at instanceof Date ? row.acquired_at : new Date(row.acquired_at); const ttlExpires = row.ttl_expires_at instanceof Date ? row.ttl_expires_at : new Date(row.ttl_expires_at); - const now = Date.now(); - // v0.41.13.0: last_refreshed_at may be NULL on pre-v98 brains that have - // the column but no acquire has happened since the migration ran. Render - // both `last_refreshed_at` and the computed delta as null so callers can - // distinguish "never observed a refresh" from "refresh fired N ms ago". const lastRefreshed = row.last_refreshed_at == null ? null - : (row.last_refreshed_at instanceof Date - ? row.last_refreshed_at - : new Date(row.last_refreshed_at)); - + : (row.last_refreshed_at instanceof Date ? row.last_refreshed_at : new Date(row.last_refreshed_at)); return { - id: lockId, + id: String(row.id ?? ''), holder_pid: Number(row.holder_pid), holder_host: String(row.holder_host ?? ''), acquired_at: acquired, @@ -370,6 +349,59 @@ export async function inspectLock(engine: BrainEngine, lockId: string): Promise< }; } +interface SelectLockRowsOpts { + /** Return only the row for this exact lock id (the `inspectLock` case). */ + lockId?: string; + /** Return only rows whose ttl_expires_at < NOW() (the `listStaleLocks` case). */ + staleOnly?: boolean; +} + +/** + * The single canonical reader for `gbrain_cycle_locks`. Engine-branches once + * (postgres / pglite) so `inspectLock`, `listStaleLocks`, and the reaper don't + * each re-roll the SELECT + Date-coercion (previously triplicated). Returns + * fully-mapped `LockSnapshot[]`; callers filter in JS (the table holds 0-2 rows + * in practice, so a no-WHERE full read is cheap and keeps the SQL trivial). + */ +async function selectLockRows(engine: BrainEngine, opts: SelectLockRowsOpts = {}): Promise { + const maybePG = engine as unknown as { sql?: (...args: unknown[]) => Promise }; + const maybePGLite = engine as unknown as { + db?: { query: (sql: string, params?: unknown[]) => Promise<{ rows: unknown[] }> }; + }; + + let rows: RawLockRow[]; + if (engine.kind === 'postgres' && maybePG.sql) { + const sql = maybePG.sql as any; + if (opts.lockId !== undefined) { + rows = await sql`SELECT id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at FROM gbrain_cycle_locks WHERE id = ${opts.lockId}`; + } else if (opts.staleOnly) { + rows = await sql`SELECT id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at FROM gbrain_cycle_locks WHERE ttl_expires_at < NOW() ORDER BY acquired_at`; + } else { + rows = await sql`SELECT id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at FROM gbrain_cycle_locks ORDER BY acquired_at`; + } + } else if (engine.kind === 'pglite' && maybePGLite.db) { + if (opts.lockId !== undefined) { + rows = (await maybePGLite.db.query(`SELECT ${LOCK_SELECT_COLS} FROM gbrain_cycle_locks WHERE id = $1`, [opts.lockId])).rows as RawLockRow[]; + } else if (opts.staleOnly) { + rows = (await maybePGLite.db.query(`SELECT ${LOCK_SELECT_COLS} FROM gbrain_cycle_locks WHERE ttl_expires_at < NOW() ORDER BY acquired_at`)).rows as RawLockRow[]; + } else { + rows = (await maybePGLite.db.query(`SELECT ${LOCK_SELECT_COLS} FROM gbrain_cycle_locks ORDER BY acquired_at`)).rows as RawLockRow[]; + } + } else { + throw new Error(`Unknown engine kind for selectLockRows: ${engine.kind}`); + } + + const now = Date.now(); + return rows + .map((r) => rowToLockSnapshot(r, now)) + .filter((s): s is LockSnapshot => s !== null); +} + +export async function inspectLock(engine: BrainEngine, lockId: string): Promise { + const rows = await selectLockRows(engine, { lockId }); + return rows[0] ?? null; +} + /** * v0.41.6.0 D3: list every lock whose TTL has expired. Used by gbrain * doctor's `stale_locks` check. The query reuses the same canonical @@ -377,55 +409,7 @@ export async function inspectLock(engine: BrainEngine, lockId: string): Promise< * UPDATE-on-conflict already trusts — no parallel heuristic. */ export async function listStaleLocks(engine: BrainEngine): Promise { - const maybePG = engine as unknown as { sql?: (...args: unknown[]) => Promise }; - const maybePGLite = engine as unknown as { - db?: { query: (sql: string, params?: unknown[]) => Promise<{ rows: unknown[] }> }; - }; - - let rows: Array<{ id?: string; holder_pid?: number; holder_host?: string; acquired_at?: Date | string; ttl_expires_at?: Date | string; last_refreshed_at?: Date | string | null }>; - - if (engine.kind === 'postgres' && maybePG.sql) { - const sql = maybePG.sql as any; - rows = await sql` - SELECT id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at - FROM gbrain_cycle_locks - WHERE ttl_expires_at < NOW() - ORDER BY acquired_at - `; - } else if (engine.kind === 'pglite' && maybePGLite.db) { - const result = await maybePGLite.db.query( - `SELECT id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at - FROM gbrain_cycle_locks - WHERE ttl_expires_at < NOW() - ORDER BY acquired_at`, - ); - rows = result.rows as typeof rows; - } else { - throw new Error(`Unknown engine kind for listStaleLocks: ${engine.kind}`); - } - - const now = Date.now(); - return rows - .filter(r => r.holder_pid !== undefined && r.acquired_at && r.ttl_expires_at) - .map(r => { - const acquired = r.acquired_at instanceof Date ? r.acquired_at : new Date(r.acquired_at!); - const ttl = r.ttl_expires_at instanceof Date ? r.ttl_expires_at : new Date(r.ttl_expires_at!); - // v0.41.13.0: last_refreshed_at may be NULL on pre-v98 brains. - const lastRefreshed = r.last_refreshed_at == null - ? null - : (r.last_refreshed_at instanceof Date ? r.last_refreshed_at : new Date(r.last_refreshed_at)); - return { - id: String(r.id ?? ''), - holder_pid: Number(r.holder_pid), - holder_host: String(r.holder_host ?? ''), - acquired_at: acquired, - ttl_expires_at: ttl, - age_ms: now - acquired.getTime(), - ttl_expired: true, - last_refreshed_at: lastRefreshed, - ms_since_last_refresh: lastRefreshed ? now - lastRefreshed.getTime() : null, - }; - }); + return selectLockRows(engine, { staleOnly: true }); } /** @@ -555,6 +539,110 @@ export async function deleteLockRowIfStale( throw new Error(`Unknown engine kind for deleteLockRowIfStale: ${engine.kind}`); } +/** + * #1972 — snapshot-matched verify-and-delete for the background reaper. + * + * Unlike `deleteLockRow` (id + holder_pid only), this ALSO pins the observed + * `acquired_at`, closing the TOCTOU window the reaper opens by reading rows + * before deleting them: between the SELECT and the DELETE, the dead holder's + * row could be replaced by a NEW holder that reused the same numeric PID + * (PID-space wraps). That new row carries a newer `acquired_at`, so the match + * fails and the DELETE is a safe no-op. + * + * Why `date_trunc('milliseconds', acquired_at) = $3` and not bare equality: + * postgres.js parses timestamptz into a JS Date (millisecond precision), so the + * snapshot we hold has already lost the microseconds that `acquired_at DEFAULT + * NOW()` writes. A bare `acquired_at = $3` would therefore NEVER match in + * production (microsecond stored value ≠ ms-truncated param) — the reaper would + * silently delete nothing. Truncating both sides to ms makes the comparison + * round-trip-safe on Postgres + PGLite, while a real takeover (seconds later) + * still differs by far more than a millisecond. + */ +export async function deleteLockRowExact( + engine: BrainEngine, + lockId: string, + holderPid: number, + acquiredAt: Date, +): Promise<{ deleted: boolean }> { + const maybePG = engine as unknown as { sql?: (...args: unknown[]) => Promise }; + const maybePGLite = engine as unknown as { + db?: { query: (sql: string, params?: unknown[]) => Promise<{ rows: unknown[] }> }; + }; + + if (engine.kind === 'postgres' && maybePG.sql) { + const sql = maybePG.sql as any; + const rows: Array<{ id: string }> = await sql` + DELETE FROM gbrain_cycle_locks + WHERE id = ${lockId} + AND holder_pid = ${holderPid} + AND date_trunc('milliseconds', acquired_at) = ${acquiredAt} + RETURNING id + `; + return { deleted: rows.length > 0 }; + } + if (engine.kind === 'pglite' && maybePGLite.db) { + const { rows } = await maybePGLite.db.query( + `DELETE FROM gbrain_cycle_locks + WHERE id = $1 + AND holder_pid = $2 + AND date_trunc('milliseconds', acquired_at) = $3 + RETURNING id`, + [lockId, holderPid, acquiredAt], + ); + return { deleted: rows.length > 0 }; + } + throw new Error(`Unknown engine kind for deleteLockRowExact: ${engine.kind}`); +} + +/** + * #1972 — host-scoped reaper for dead-holder locks. Closes the gap where a sync + * (or cycle) that crashed (OOM, recycle, SIGKILL) strands its lock row until + * something contends for it — a low-traffic source could look "syncing" for + * hours. `tryAcquireDbLock` only reclaims on contention; this is the background + * sweep. Intended to run at cycle start (and under `gbrain doctor --fix` for + * no-autopilot brains). + * + * Scoped to the `gbrain-sync:*` and `gbrain-cycle`/`gbrain-cycle:*` namespaces + * ONLY. `gbrain_cycle_locks` is shared by enrich, the minion supervisor, + * reindex, schema-pack, and elections (`tryWithDbElection`) — a blanket sweep + * would change their TTL-failover timing. Those keep their existing + * on-contention + TTL behavior. + * + * reapDeadHolderLocks (host-scoped, sync/cycle namespaces only) + * selectLockRows → for each row: + * ├─ id not in sync/cycle namespace ───────→ KEEP (blast-radius scope) + * ├─ holder_host != thisHost ──────────────→ KEEP (cross_host; can't probe) + * ├─ kill(pid,0) == alive / EPERM ─────────→ KEEP (live or not-ours) + * ├─ ESRCH && age < 60s grace ─────────────→ KEEP (PID-reuse defense) + * └─ ESRCH && age ≥ 60s grace ─────────────→ deleteLockRowExact(id, pid, acquired_at) + * (snapshot-matched: reused-PID + * fresh row has newer acquired_at → no-op) + */ +function isReapableNamespace(lockId: string): boolean { + return ( + lockId === 'gbrain-cycle' || + lockId.startsWith('gbrain-cycle:') || + lockId.startsWith('gbrain-sync:') + ); +} + +export async function reapDeadHolderLocks( + engine: BrainEngine, + opts: HolderLivenessOpts = {}, +): Promise<{ reaped: number; reapedIds: string[] }> { + const rows = await selectLockRows(engine); + const reapedIds: string[] = []; + for (const s of rows) { + if (!isReapableNamespace(s.id)) continue; + // isHolderDeadLocally combines same-host + ESRCH + the 60s reuse grace, + // so pure-PID-liveness (which PID-space wrap defeats) is never used alone. + if (!isHolderDeadLocally(s.holder_pid, s.holder_host, s.age_ms, opts)) continue; + const { deleted } = await deleteLockRowExact(engine, s.id, s.holder_pid, s.acquired_at); + if (deleted) reapedIds.push(s.id); + } + return { reaped: reapedIds.length, reapedIds }; +} + /** * v0.40 (Federated Sync v2): per-source sync lock helper. * diff --git a/test/db-lock-reap.test.ts b/test/db-lock-reap.test.ts new file mode 100644 index 000000000..2b32da5a1 --- /dev/null +++ b/test/db-lock-reap.test.ts @@ -0,0 +1,162 @@ +/** + * #1972 — host-scoped reaper for dead-holder sync/cycle locks. + * + * Covers: + * - reapDeadHolderLocks: namespace scope (sync/cycle only, NOT election/etc), + * same-host dead-PID reaped regardless of TTL, live/cross-host/within-grace + * kept. Uses the injectable process.kill seam so it's deterministic. + * - deleteLockRowExact: snapshot-matched delete (the TOCTOU defense) — a + * non-matching acquired_at is a no-op, the matching one deletes. + */ + +import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test'; +import { hostname } from 'os'; +import { PGLiteEngine } from '../src/core/pglite-engine.ts'; +import { + reapDeadHolderLocks, + deleteLockRowExact, + inspectLock, + HOLDER_TAKEOVER_GRACE_MS, + type HolderLivenessOpts, +} from '../src/core/db-lock.ts'; + +let engine: PGLiteEngine; + +beforeAll(async () => { + engine = new PGLiteEngine(); + await engine.connect({ database_url: '' }); + await engine.initSchema(); +}); + +afterAll(async () => { + await engine.disconnect(); +}); + +beforeEach(async () => { + await engine.executeRaw(`DELETE FROM gbrain_cycle_locks`); +}); + +const LOCAL = hostname(); +const OLD_S = Math.ceil(HOLDER_TAKEOVER_GRACE_MS / 1000) + 60; // comfortably past grace +const YOUNG_S = 5; + +/** Insert a lock row with a given age + namespace. ttlFuture controls whether + * the row is TTL-expired (to prove the reaper reaps dead holders regardless). */ +async function seedLock( + id: string, + holderPid: number, + holderHost: string, + ageSeconds: number, + ttlFuture = true, +) { + const ttl = ttlFuture ? `NOW() + INTERVAL '10 minutes'` : `NOW() - INTERVAL '1 minute'`; + await engine.executeRaw( + `INSERT INTO gbrain_cycle_locks (id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at) + VALUES ($1, $2, $3, NOW() - ($4 || ' seconds')::interval, ${ttl}, NOW() - ($4 || ' seconds')::interval)`, + [id, holderPid, holderHost, String(ageSeconds)], + ); +} + +/** process.kill seam: every pid is dead (ESRCH) except those in `live`. */ +function killSeam(live: Set): HolderLivenessOpts { + return { + localHost: LOCAL, + processKill: (pid: number) => { + if (live.has(pid)) return; // alive + const e = new Error('no such process') as NodeJS.ErrnoException; + e.code = 'ESRCH'; + throw e; + }, + }; +} + +async function lockIds(): Promise { + const rows = await engine.executeRaw<{ id: string }>(`SELECT id FROM gbrain_cycle_locks ORDER BY id`); + return rows.map(r => r.id); +} + +describe('reapDeadHolderLocks', () => { + test('reaps same-host dead-PID sync + cycle locks (regardless of TTL)', async () => { + await seedLock('gbrain-sync:src-a', 900001, LOCAL, OLD_S, /*ttlFuture*/ true); // dead, TTL NOT expired + await seedLock('gbrain-cycle', 900002, LOCAL, OLD_S, false); // dead, TTL expired + await seedLock('gbrain-cycle:src-b', 900003, LOCAL, OLD_S, true); // dead, TTL NOT expired + + const { reaped, reapedIds } = await reapDeadHolderLocks(engine, killSeam(new Set())); + expect(reaped).toBe(3); + expect(reapedIds.sort()).toEqual(['gbrain-cycle', 'gbrain-cycle:src-b', 'gbrain-sync:src-a']); + expect(await lockIds()).toEqual([]); + }); + + test('keeps a live same-host holder', async () => { + await seedLock('gbrain-sync:src-live', process.pid, LOCAL, OLD_S); + const { reaped } = await reapDeadHolderLocks(engine, killSeam(new Set([process.pid]))); + expect(reaped).toBe(0); + expect(await lockIds()).toEqual(['gbrain-sync:src-live']); + }); + + test('keeps a dead holder still within the PID-reuse grace window', async () => { + await seedLock('gbrain-sync:src-young', 900010, LOCAL, YOUNG_S); + const { reaped } = await reapDeadHolderLocks(engine, killSeam(new Set())); + expect(reaped).toBe(0); + expect(await lockIds()).toEqual(['gbrain-sync:src-young']); + }); + + test('keeps a cross-host holder (cannot probe a remote PID)', async () => { + await seedLock('gbrain-sync:src-xhost', 900011, 'a-different-host', OLD_S); + const { reaped } = await reapDeadHolderLocks(engine, killSeam(new Set())); + expect(reaped).toBe(0); + expect(await lockIds()).toEqual(['gbrain-sync:src-xhost']); + }); + + test('NEVER reaps a non-sync/cycle namespace, even with a dead PID (blast radius)', async () => { + await seedLock('gbrain-election:leader', 900020, LOCAL, OLD_S); + await seedLock('some-other-lock', 900021, LOCAL, OLD_S); + const { reaped } = await reapDeadHolderLocks(engine, killSeam(new Set())); + expect(reaped).toBe(0); + expect(await lockIds()).toEqual(['gbrain-election:leader', 'some-other-lock']); + }); + + test('empty table → {reaped:0}', async () => { + const r = await reapDeadHolderLocks(engine, killSeam(new Set())); + expect(r).toEqual({ reaped: 0, reapedIds: [] }); + }); + + test('mixed set: reaps only the eligible rows', async () => { + await seedLock('gbrain-sync:dead', 900030, LOCAL, OLD_S); // reap + await seedLock('gbrain-sync:live', process.pid, LOCAL, OLD_S); // keep (live) + await seedLock('gbrain-cycle:xhost', 900031, 'other', OLD_S); // keep (cross-host) + await seedLock('gbrain-election:x', 900032, LOCAL, OLD_S); // keep (namespace) + const { reaped, reapedIds } = await reapDeadHolderLocks(engine, killSeam(new Set([process.pid]))); + expect(reaped).toBe(1); + expect(reapedIds).toEqual(['gbrain-sync:dead']); + expect(await lockIds()).toEqual(['gbrain-cycle:xhost', 'gbrain-election:x', 'gbrain-sync:live']); + }); +}); + +describe('deleteLockRowExact (snapshot-matched, TOCTOU defense)', () => { + test('no-op when acquired_at does not match; deletes when it does', async () => { + await seedLock('gbrain-sync:exact', 900040, LOCAL, OLD_S); + const snap = await inspectLock(engine, 'gbrain-sync:exact'); + expect(snap).not.toBeNull(); + + // Simulate a reused-PID takeover: same id + pid, but a different (newer) + // acquired_at than the one the reaper snapshotted → must NOT delete. + const wrong = new Date(snap!.acquired_at.getTime() + 3_600_000); + const miss = await deleteLockRowExact(engine, 'gbrain-sync:exact', 900040, wrong); + expect(miss.deleted).toBe(false); + expect(await lockIds()).toEqual(['gbrain-sync:exact']); + + // The matching snapshot deletes. + const hit = await deleteLockRowExact(engine, 'gbrain-sync:exact', 900040, snap!.acquired_at); + expect(hit.deleted).toBe(true); + expect(await lockIds()).toEqual([]); + }); + + test('no-op when holder_pid does not match', async () => { + await seedLock('gbrain-sync:pidguard', 900050, LOCAL, OLD_S); + const snap = await inspectLock(engine, 'gbrain-sync:pidguard'); + const res = await deleteLockRowExact(engine, 'gbrain-sync:pidguard', 111111, snap!.acquired_at); + expect(res.deleted).toBe(false); + expect(await lockIds()).toEqual(['gbrain-sync:pidguard']); + }); +});