From 6cf8d8d66c86d3a5421715aa2a23657d39624b85 Mon Sep 17 00:00:00 2001
From: Khaja Nazimuddin <34912639+Nazim22@users.noreply.github.com>
Date: Wed, 22 Jul 2026 20:46:54 -0500
Subject: [PATCH] fix(extract): clear pre-version-bump pages in extract --stale
 (#1791)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`extractStaleFromDB` stamped `links_extracted_at` with each page's read
`updated_at` (the D4 race-fix). But the stale predicate also flags
`links_extracted_at < LINK_EXTRACTOR_VERSION_TS`. Any page last edited
BEFORE the version timestamp got stamped below the threshold, so the
version arm re-flagged it stale on every run — an infinite re-extract
loop that never cleared the lag.

Since the v112 watermark column ships with no backfill, every
pre-existing page starts stale, and most pre-date the version bump. In
practice this left ~97% of pages permanently stale: `extract --stale`
reported "done" each run but `links_extraction_lag` never dropped.

Fix: stamp `GREATEST(read updated_at, versionTs)`. Old pages lift to the
threshold so the version arm clears; a real future edit still advances
`updated_at` past the stamp, so the CDX-1 edited-after-stamp race
protection is preserved.

Adds a regression test: a page with `updated_at` before
LINK_EXTRACTOR_VERSION_TS must clear after extract AND stay clear on a
second run (the existing tests only used now()-dated pages, so the
old-page case was uncovered).

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
---
 src/commands/extract.ts    | 13 ++++++++++++-
 test/extract-stale.test.ts | 26 ++++++++++++++++++++++++++
 2 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/src/commands/extract.ts b/src/commands/extract.ts
index db232ec51..98176e317 100644
--- a/src/commands/extract.ts
+++ b/src/commands/extract.ts
@@ -1743,7 +1743,18 @@ export async function extractStaleFromDB(
       // `page.updated_at.toISOString()` — the JS Date is ms-truncated, so the
       // µs-precision DB updated_at stayed strictly greater and the page never
       // cleared on Postgres. Stamping the exact value makes them equal.
-      processedRefs.push({ slug: page.slug, source_id: page.source_id, extractedAt: page.updated_at_iso });
+      //
+      // BUT the stamp must also clear the version-staleness clause
+      // (`links_extracted_at < versionTs`). A page whose updated_at predates
+      // versionTs would otherwise be stamped below the threshold and read as
+      // stale forever — a permanent re-extract loop that never clears the lag.
+      // GREATEST(updated_at, versionTs) preserves the race semantics (a real
+      // future edit advances updated_at > versionTs >= stamp → re-extracts)
+      // while lifting old pages to the threshold so they clear.
+      const stampIso = page.updated_at.getTime() >= Date.parse(versionTs)
+        ? page.updated_at_iso
+        : versionTs;
+      processedRefs.push({ slug: page.slug, source_id: page.source_id, extractedAt: stampIso });
     }
 
     // Flush NON-swallowing (CDX-4): a throw here propagates out of the sweep so
diff --git a/test/extract-stale.test.ts b/test/extract-stale.test.ts
index a725e309b..f6db5ff18 100644
--- a/test/extract-stale.test.ts
+++ b/test/extract-stale.test.ts
@@ -209,6 +209,32 @@ describe('gbrain extract --stale', () => {
     expect(usRows[0]?.eq).toBe(true);
   });
 
+  test('REGRESSION: page with updated_at BEFORE LINK_EXTRACTOR_VERSION_TS clears (no permanent-stale loop)', async () => {
+    // The v112 watermark column ships with no backfill, so every pre-existing
+    // page starts NULL-stale — and most pre-date the version bump. Pre-fix,
+    // extractStaleFromDB stamped links_extracted_at = read updated_at; for a
+    // page edited before LINK_EXTRACTOR_VERSION_TS the stamp landed BELOW the
+    // version threshold, so the version arm (links_extracted_at < versionTs)
+    // re-flagged it stale forever — an infinite re-extract loop that never
+    // cleared the lag (observed: 97% of pages stuck permanently).
+    await engine.putPage('people/alice', personPage('Alice'));
+    await engine.putPage('companies/acme', companyPage('Acme', '[Alice](people/alice) leads [Acme](companies/acme).'));
+    // Backdate every page to BEFORE the extractor version timestamp.
+    await engine.executeRaw(`UPDATE pages SET updated_at = '2020-01-01T00:00:00Z'`);
+    expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(2);
+
+    await runExtract(engine, ['--stale']);
+    // Fixed: stamp = GREATEST(read updated_at, versionTs) → lifts old pages to
+    // the threshold so the version arm clears, while a real future edit still
+    // advances updated_at past the stamp (CDX-1 race protection preserved).
+    expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(0);
+
+    // Second run must ALSO find 0 — the defining symptom of the bug was that it
+    // never converged.
+    await runExtract(engine, ['--stale']);
+    expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(0);
+  });
+
   test('CDX-4 (D2): a link-flush throw aborts the sweep and leaves pages UNSTAMPED', async () => {
     await engine.putPage('people/alice', personPage('Alice'));
     await engine.putPage('companies/acme', companyPage('Acme', '[Alice](people/alice) founded [Acme](companies/acme).'));