diff --git a/src/commands/extract.ts b/src/commands/extract.ts index db232ec51..98176e317 100644 --- a/src/commands/extract.ts +++ b/src/commands/extract.ts @@ -1743,7 +1743,18 @@ export async function extractStaleFromDB( // `page.updated_at.toISOString()` — the JS Date is ms-truncated, so the // µs-precision DB updated_at stayed strictly greater and the page never // cleared on Postgres. Stamping the exact value makes them equal. - processedRefs.push({ slug: page.slug, source_id: page.source_id, extractedAt: page.updated_at_iso }); + // + // BUT the stamp must also clear the version-staleness clause + // (`links_extracted_at < versionTs`). A page whose updated_at predates + // versionTs would otherwise be stamped below the threshold and read as + // stale forever — a permanent re-extract loop that never clears the lag. + // GREATEST(updated_at, versionTs) preserves the race semantics (a real + // future edit advances updated_at > versionTs >= stamp → re-extracts) + // while lifting old pages to the threshold so they clear. + const stampIso = page.updated_at.getTime() >= Date.parse(versionTs) + ? page.updated_at_iso + : versionTs; + processedRefs.push({ slug: page.slug, source_id: page.source_id, extractedAt: stampIso }); } // Flush NON-swallowing (CDX-4): a throw here propagates out of the sweep so diff --git a/test/extract-stale.test.ts b/test/extract-stale.test.ts index a725e309b..f6db5ff18 100644 --- a/test/extract-stale.test.ts +++ b/test/extract-stale.test.ts @@ -209,6 +209,32 @@ describe('gbrain extract --stale', () => { expect(usRows[0]?.eq).toBe(true); }); + test('REGRESSION: page with updated_at BEFORE LINK_EXTRACTOR_VERSION_TS clears (no permanent-stale loop)', async () => { + // The v112 watermark column ships with no backfill, so every pre-existing + // page starts NULL-stale — and most pre-date the version bump. Pre-fix, + // extractStaleFromDB stamped links_extracted_at = read updated_at; for a + // page edited before LINK_EXTRACTOR_VERSION_TS the stamp landed BELOW the + // version threshold, so the version arm (links_extracted_at < versionTs) + // re-flagged it stale forever — an infinite re-extract loop that never + // cleared the lag (observed: 97% of pages stuck permanently). + await engine.putPage('people/alice', personPage('Alice')); + await engine.putPage('companies/acme', companyPage('Acme', '[Alice](people/alice) leads [Acme](companies/acme).')); + // Backdate every page to BEFORE the extractor version timestamp. + await engine.executeRaw(`UPDATE pages SET updated_at = '2020-01-01T00:00:00Z'`); + expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(2); + + await runExtract(engine, ['--stale']); + // Fixed: stamp = GREATEST(read updated_at, versionTs) → lifts old pages to + // the threshold so the version arm clears, while a real future edit still + // advances updated_at past the stamp (CDX-1 race protection preserved). + expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(0); + + // Second run must ALSO find 0 — the defining symptom of the bug was that it + // never converged. + await runExtract(engine, ['--stale']); + expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(0); + }); + test('CDX-4 (D2): a link-flush throw aborts the sweep and leaves pages UNSTAMPED', async () => { await engine.putPage('people/alice', personPage('Alice')); await engine.putPage('companies/acme', companyPage('Acme', '[Alice](people/alice) founded [Acme](companies/acme).'));