From 8dc3310483cc60da80dfedeca1b975133210a46e Mon Sep 17 00:00:00 2001
From: Time Attakc <89218912+time-attack@users.noreply.github.com>
Date: Thu, 23 Jul 2026 12:42:40 -0700
Subject: [PATCH] fix(dream): stamp incremental extraction watermark (#2636)
 (#3115)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The Dream cycle disables sync's inline extraction and routes changed
slugs through extractForSlugs, which flushed link/timeline batches but
never stamped links_extracted_at — so incrementally extracted pages
stayed permanently visible to `extract --stale` / doctor.

Collect processedRefs per successfully processed page and stamp them
via stampExtracted (best-effort) after both batch flushes, non-dry-run
mode 'all' only. Source-id threading from the original PR #2637 already
landed on master via #1503/#1747, so this rebase carries only the
missing watermark stamp plus regression tests.

Takeover of #2637.

Co-authored-by: Garry Tan <garrytan@gmail.com>
Co-authored-by: JavanC <JavanC@users.noreply.github.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
---
 src/commands/extract.ts          | 12 +++++++++
 test/extract-incremental.test.ts | 45 ++++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+)

diff --git a/src/commands/extract.ts b/src/commands/extract.ts
index 928f1fc7a..b2aabc5fd 100644
--- a/src/commands/extract.ts
+++ b/src/commands/extract.ts
@@ -1025,6 +1025,10 @@ async function extractForSlugs(
   let linksCreated = 0;
   let timelineCreated = 0;
   let pagesProcessed = 0;
+  // #2636: successfully processed pages get their extraction watermark
+  // stamped after the final flush (mode 'all' only — a partial-mode run
+  // hasn't done the full extraction the watermark asserts).
+  const processedRefs: Array<{ slug: string; source_id: string }> = [];
 
   // Issue #972: read the basename flag once per extract run.
   const globalBasename = await isGlobalBasenameEnabled(engine);
@@ -1113,6 +1117,7 @@ async function extractForSlugs(
         }
 
         pagesProcessed++;
+        if (!dryRun) processedRefs.push({ slug, source_id: sourceId ?? 'default' });
       } catch { /* skip unreadable */ }
       progress.tick(1);
     },
@@ -1120,6 +1125,13 @@ async function extractForSlugs(
 
   await flushLinks();
   await flushTimeline();
+  // #2636: the Dream cycle disables sync's inline extraction and routes
+  // changed slugs through this incremental path — without a stamp here,
+  // those pages never get links_extracted_at and stay permanently visible
+  // to `extract --stale` / doctor. Stamp only after BOTH batches flushed.
+  if (!dryRun && mode === 'all') {
+    await stampExtracted(engine, processedRefs);
+  }
   progress.finish();
 
   if (!jsonMode) {
diff --git a/test/extract-incremental.test.ts b/test/extract-incremental.test.ts
index a4453d77c..dc47069f6 100644
--- a/test/extract-incremental.test.ts
+++ b/test/extract-incremental.test.ts
@@ -60,6 +60,51 @@ async function seedPage(slug: string, body: string): Promise<void> {
 }
 
 describe('runExtractCore — incremental cycle path (#417)', () => {
+  test('Dream incremental all-mode stamps the source-scoped extraction watermark (#2636)', async () => {
+    await engine.executeRaw(
+      `INSERT INTO sources (id, name, local_path) VALUES ($1, $2, $3)`,
+      ['repo-a', 'repo-a', tempDir],
+    );
+    await engine.putPage('people/alice-example', {
+      type: 'person',
+      title: 'alice-example',
+      compiled_truth: '# alice',
+      timeline: '',
+      frontmatter: {},
+      content_hash: 'h',
+    }, { sourceId: 'repo-a' });
+    writeFileSync(join(tempDir, 'people/alice-example.md'), '# alice');
+
+    await runExtractCore(engine as unknown as BrainEngine, {
+      mode: 'all',
+      dir: tempDir,
+      slugs: ['people/alice-example'],
+      sourceId: 'repo-a',
+    });
+
+    const rows = await engine.executeRaw<{ links_extracted_at: string | null }>(
+      `SELECT links_extracted_at FROM pages WHERE slug = $1 AND source_id = $2`,
+      ['people/alice-example', 'repo-a'],
+    );
+    expect(rows[0]?.links_extracted_at).not.toBeNull();
+    expect(await engine.countStalePagesForExtraction({ sourceId: 'repo-a' })).toBe(0);
+  });
+
+  test('Dream incremental dry-run does NOT stamp the watermark', async () => {
+    await seedPage('people/alice-example', '# alice');
+    await runExtractCore(engine as unknown as BrainEngine, {
+      mode: 'all',
+      dir: tempDir,
+      slugs: ['people/alice-example'],
+      dryRun: true,
+    });
+    const rows = await engine.executeRaw<{ links_extracted_at: string | null }>(
+      `SELECT links_extracted_at FROM pages WHERE slug = $1`,
+      ['people/alice-example'],
+    );
+    expect(rows[0]?.links_extracted_at ?? null).toBeNull();
+  });
+
   test('1. slugs: [] returns immediately with zero counts (early-return path)', async () => {
     await seedPage('people/alice-example', '# alice');
     const result = await runExtractCore(engine as unknown as BrainEngine, {