fix(takes): bootstrap runs progress through the corpus instead of rescanning the newest slice (#2638)

extractTakesFromPages selected pages by updated_at DESC + LIMIT with no
exclusion of pages that already hold takes. The CLI clamps --max-pages to
1000, so on a corpus larger than one run every re-run rescanned the same
most-recent 1000: the older tail could never be bootstrapped, and each
rescan re-spent Haiku budget producing upsert-identical rows. Seen live on
a 2,311-eligible-page brain — a second run would have covered 0 new pages.

Covered pages are now skipped by default (NOT EXISTS on takes.page_id), so
repeat runs sweep a large corpus in slices; --include-covered restores the
full rescan for refresh use. Usage text documents both plus the 1000 clamp.


Claude-Session: https://claude.ai/code/session_01FQgByq4aqQq2PP8UHCdnfk

Co-authored-by: Paolo Belcastro <p3ob7o@users.noreply.github.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

src/commands/takes.ts

@@ -605,7 +605,8 @@ async function cmdExtract(engine: BrainEngine, rest: string[]): Promise<void> {
   const sub = rest[0];
   if (sub !== '--from-pages') {
     process.stderr.write(
-      'Usage: gbrain takes extract --from-pages [--yes] [--dry-run] [--source-id <id>] [--max-pages N] [--holder <name>]\n',
+      'Usage: gbrain takes extract --from-pages [--yes] [--dry-run] [--source-id <id>] [--max-pages N (clamped to 1000)] [--include-covered] [--holder <name>]\n' +
+      'Runs progress: pages that already hold takes are skipped, so repeat runs sweep a large corpus in slices. --include-covered rescans everything (refresh).\n',
     );
     process.exit(1);
   }
@@ -618,6 +619,7 @@ async function cmdExtract(engine: BrainEngine, rest: string[]): Promise<void> {
   const maxPages = maxPagesRaw ? Math.max(1, Math.min(1000, parseInt(maxPagesRaw, 10) || 50)) : 50;
   const holderIdx = rest.indexOf('--holder');
   const holder = holderIdx >= 0 ? rest[holderIdx + 1] : 'system';
+  const includeCovered = rest.includes('--include-covered');
 
   // A12 consent gate.
   const bootstrapEnabledCfg = await engine.getConfig('takes.bootstrap_enabled');
@@ -642,6 +644,7 @@ async function cmdExtract(engine: BrainEngine, rest: string[]): Promise<void> {
     dryRun,
     sourceIdFilter,
     maxPages,
+    includeCovered,
     holder,
   });
   if (result.llm_unavailable) {

src/core/extract-takes-from-pages.ts

@@ -46,6 +46,13 @@ export interface ExtractTakesFromPagesOpts {
   sourceIdFilter?: string;
   /** Max pages to classify per run (caps cost). Default 50. */
   maxPages?: number;
+  /**
+   * Also rescan pages that already hold takes (refresh semantics).
+   * Default false: bootstrap runs skip covered pages, so repeated runs
+   * PROGRESS through a corpus larger than one run's cap instead of
+   * rescanning the same most-recently-updated slice forever.
+   */
+  includeCovered?: boolean;
   /** Owner identifier for the inserted takes. Default 'system'. */
   holder?: string;
   /** Model override; defaults to facts.extraction_model. */
@@ -132,12 +139,21 @@ export async function extractTakesFromPages(
   // Fetch eligible pages. Order by updated_at DESC so recently-edited
   // pages get bootstrapped first.
   const typesList = ALLOWED_PAGE_TYPES.map((t) => `'${t}'`).join(', ');
+  // Bootstrap progression: skip pages that already hold takes (opt out via
+  // includeCovered). Without this, the updated_at-DESC + LIMIT selection made
+  // every re-run rescan the same most-recent slice — a corpus larger than one
+  // run's cap could never be fully bootstrapped (and each rescan re-spent LLM
+  // budget on covered pages for upsert-identical rows).
+  const coveredFilter = opts.includeCovered
+    ? ''
+    : `AND NOT EXISTS (SELECT 1 FROM takes t WHERE t.page_id = pages.id)`;
   const pages = await engine.executeRaw<PageRow>(
     `SELECT id, slug, source_id, type, compiled_truth, updated_at
        FROM pages
       WHERE type IN (${typesList})
         AND deleted_at IS NULL
         AND length(COALESCE(compiled_truth, '')) > 200
+        ${coveredFilter}
         ${sourceFilter}
       ORDER BY updated_at DESC
       LIMIT ${maxPages}`,

test/extract-takes-from-pages-progression.test.ts

@@ -0,0 +1,87 @@
+/**
+ * Bootstrap progression regression test.
+ *
+ * extractTakesFromPages selected pages by updated_at DESC + LIMIT with no
+ * exclusion of pages that already hold takes — so on a corpus larger than
+ * one run's cap (the CLI clamps --max-pages to 1000), every re-run rescanned
+ * the same most-recent slice: the older tail could never be bootstrapped,
+ * and each rescan re-spent LLM budget producing upsert-identical rows.
+ * Seen live on a 2,311-eligible-page brain where the second run would have
+ * covered 0 new pages.
+ *
+ * Pins: covered pages are skipped by default (runs progress), and
+ * includeCovered restores the full rescan (refresh semantics).
+ */
+import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
+import { PGLiteEngine } from '../src/core/pglite-engine.ts';
+import {
+  configureGateway,
+  resetGateway,
+  __setChatTransportForTests,
+} from '../src/core/ai/gateway.ts';
+import { extractTakesFromPages } from '../src/core/extract-takes-from-pages.ts';
+
+let engine: PGLiteEngine;
+
+beforeAll(async () => {
+  engine = new PGLiteEngine();
+  await engine.connect({});
+  await engine.initSchema();
+
+  configureGateway({
+    chat_model: 'anthropic:claude-haiku-4-5-20251001',
+    env: { ANTHROPIC_API_KEY: 'sk-ant-test-takes-progression' },
+  });
+  __setChatTransportForTests(async () => ({
+    text: '[{"claim":"a stubbed claim","kind":"take","weight":0.7}]',
+    blocks: [{ type: 'text' as const, text: '[{"claim":"a stubbed claim","kind":"take","weight":0.7}]' }],
+    stopReason: 'end' as const,
+    usage: { input_tokens: 1, output_tokens: 1, cache_read_tokens: 0, cache_creation_tokens: 0 },
+    model: 'anthropic:claude-haiku-4-5-20251001',
+    providerId: 'anthropic',
+  }));
+
+  const body = 'An opinion-bearing body long enough to clear the 200-char eligibility floor. '.repeat(5);
+  await engine.putPage('concepts/progression-a', {
+    type: 'concept', title: 'A', compiled_truth: body, frontmatter: {},
+  });
+  await engine.putPage('concepts/progression-b', {
+    type: 'concept', title: 'B', compiled_truth: body, frontmatter: {},
+  });
+});
+
+afterAll(async () => {
+  __setChatTransportForTests(null);
+  resetGateway();
+  await engine.disconnect();
+});
+
+describe('extractTakesFromPages — bootstrap progression', () => {
+  test('first run covers the eligible pages', async () => {
+    const r1 = await extractTakesFromPages(engine, { bootstrapEnabled: true, maxPages: 50 });
+    expect(r1.pages_scanned).toBe(2);
+    expect(r1.claims_extracted).toBe(2);
+  });
+
+  test('second run skips covered pages — repeat runs progress instead of rescanning', async () => {
+    const r2 = await extractTakesFromPages(engine, { bootstrapEnabled: true, maxPages: 50 });
+    expect(r2.pages_scanned).toBe(0);
+    expect(r2.claims_extracted).toBe(0);
+  });
+
+  test('a page added after the first run is picked up (progression, not a frozen set)', async () => {
+    const body = 'Another opinion-bearing body long enough to clear the eligibility floor. '.repeat(5);
+    await engine.putPage('concepts/progression-c', {
+      type: 'concept', title: 'C', compiled_truth: body, frontmatter: {},
+    });
+    const r3 = await extractTakesFromPages(engine, { bootstrapEnabled: true, maxPages: 50 });
+    expect(r3.pages_scanned).toBe(1);
+  });
+
+  test('includeCovered rescans everything (refresh semantics)', async () => {
+    const r4 = await extractTakesFromPages(engine, {
+      bootstrapEnabled: true, maxPages: 50, includeCovered: true,
+    });
+    expect(r4.pages_scanned).toBe(3);
+  });
+});