diff --git a/src/commands/takes.ts b/src/commands/takes.ts index 4e2c30b80..5dd24911e 100644 --- a/src/commands/takes.ts +++ b/src/commands/takes.ts @@ -605,7 +605,8 @@ async function cmdExtract(engine: BrainEngine, rest: string[]): Promise { const sub = rest[0]; if (sub !== '--from-pages') { process.stderr.write( - 'Usage: gbrain takes extract --from-pages [--yes] [--dry-run] [--source-id ] [--max-pages N] [--holder ]\n', + 'Usage: gbrain takes extract --from-pages [--yes] [--dry-run] [--source-id ] [--max-pages N (clamped to 1000)] [--include-covered] [--holder ]\n' + + 'Runs progress: pages that already hold takes are skipped, so repeat runs sweep a large corpus in slices. --include-covered rescans everything (refresh).\n', ); process.exit(1); } @@ -618,6 +619,7 @@ async function cmdExtract(engine: BrainEngine, rest: string[]): Promise { const maxPages = maxPagesRaw ? Math.max(1, Math.min(1000, parseInt(maxPagesRaw, 10) || 50)) : 50; const holderIdx = rest.indexOf('--holder'); const holder = holderIdx >= 0 ? rest[holderIdx + 1] : 'system'; + const includeCovered = rest.includes('--include-covered'); // A12 consent gate. const bootstrapEnabledCfg = await engine.getConfig('takes.bootstrap_enabled'); @@ -642,6 +644,7 @@ async function cmdExtract(engine: BrainEngine, rest: string[]): Promise { dryRun, sourceIdFilter, maxPages, + includeCovered, holder, }); if (result.llm_unavailable) { diff --git a/src/core/extract-takes-from-pages.ts b/src/core/extract-takes-from-pages.ts index 931eb0624..784f152ac 100644 --- a/src/core/extract-takes-from-pages.ts +++ b/src/core/extract-takes-from-pages.ts @@ -46,6 +46,13 @@ export interface ExtractTakesFromPagesOpts { sourceIdFilter?: string; /** Max pages to classify per run (caps cost). Default 50. */ maxPages?: number; + /** + * Also rescan pages that already hold takes (refresh semantics). + * Default false: bootstrap runs skip covered pages, so repeated runs + * PROGRESS through a corpus larger than one run's cap instead of + * rescanning the same most-recently-updated slice forever. + */ + includeCovered?: boolean; /** Owner identifier for the inserted takes. Default 'system'. */ holder?: string; /** Model override; defaults to facts.extraction_model. */ @@ -132,12 +139,21 @@ export async function extractTakesFromPages( // Fetch eligible pages. Order by updated_at DESC so recently-edited // pages get bootstrapped first. const typesList = ALLOWED_PAGE_TYPES.map((t) => `'${t}'`).join(', '); + // Bootstrap progression: skip pages that already hold takes (opt out via + // includeCovered). Without this, the updated_at-DESC + LIMIT selection made + // every re-run rescan the same most-recent slice — a corpus larger than one + // run's cap could never be fully bootstrapped (and each rescan re-spent LLM + // budget on covered pages for upsert-identical rows). + const coveredFilter = opts.includeCovered + ? '' + : `AND NOT EXISTS (SELECT 1 FROM takes t WHERE t.page_id = pages.id)`; const pages = await engine.executeRaw( `SELECT id, slug, source_id, type, compiled_truth, updated_at FROM pages WHERE type IN (${typesList}) AND deleted_at IS NULL AND length(COALESCE(compiled_truth, '')) > 200 + ${coveredFilter} ${sourceFilter} ORDER BY updated_at DESC LIMIT ${maxPages}`, diff --git a/test/extract-takes-from-pages-progression.test.ts b/test/extract-takes-from-pages-progression.test.ts new file mode 100644 index 000000000..f8bc2dbc2 --- /dev/null +++ b/test/extract-takes-from-pages-progression.test.ts @@ -0,0 +1,87 @@ +/** + * Bootstrap progression regression test. + * + * extractTakesFromPages selected pages by updated_at DESC + LIMIT with no + * exclusion of pages that already hold takes — so on a corpus larger than + * one run's cap (the CLI clamps --max-pages to 1000), every re-run rescanned + * the same most-recent slice: the older tail could never be bootstrapped, + * and each rescan re-spent LLM budget producing upsert-identical rows. + * Seen live on a 2,311-eligible-page brain where the second run would have + * covered 0 new pages. + * + * Pins: covered pages are skipped by default (runs progress), and + * includeCovered restores the full rescan (refresh semantics). + */ +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { PGLiteEngine } from '../src/core/pglite-engine.ts'; +import { + configureGateway, + resetGateway, + __setChatTransportForTests, +} from '../src/core/ai/gateway.ts'; +import { extractTakesFromPages } from '../src/core/extract-takes-from-pages.ts'; + +let engine: PGLiteEngine; + +beforeAll(async () => { + engine = new PGLiteEngine(); + await engine.connect({}); + await engine.initSchema(); + + configureGateway({ + chat_model: 'anthropic:claude-haiku-4-5-20251001', + env: { ANTHROPIC_API_KEY: 'sk-ant-test-takes-progression' }, + }); + __setChatTransportForTests(async () => ({ + text: '[{"claim":"a stubbed claim","kind":"take","weight":0.7}]', + blocks: [{ type: 'text' as const, text: '[{"claim":"a stubbed claim","kind":"take","weight":0.7}]' }], + stopReason: 'end' as const, + usage: { input_tokens: 1, output_tokens: 1, cache_read_tokens: 0, cache_creation_tokens: 0 }, + model: 'anthropic:claude-haiku-4-5-20251001', + providerId: 'anthropic', + })); + + const body = 'An opinion-bearing body long enough to clear the 200-char eligibility floor. '.repeat(5); + await engine.putPage('concepts/progression-a', { + type: 'concept', title: 'A', compiled_truth: body, frontmatter: {}, + }); + await engine.putPage('concepts/progression-b', { + type: 'concept', title: 'B', compiled_truth: body, frontmatter: {}, + }); +}); + +afterAll(async () => { + __setChatTransportForTests(null); + resetGateway(); + await engine.disconnect(); +}); + +describe('extractTakesFromPages — bootstrap progression', () => { + test('first run covers the eligible pages', async () => { + const r1 = await extractTakesFromPages(engine, { bootstrapEnabled: true, maxPages: 50 }); + expect(r1.pages_scanned).toBe(2); + expect(r1.claims_extracted).toBe(2); + }); + + test('second run skips covered pages — repeat runs progress instead of rescanning', async () => { + const r2 = await extractTakesFromPages(engine, { bootstrapEnabled: true, maxPages: 50 }); + expect(r2.pages_scanned).toBe(0); + expect(r2.claims_extracted).toBe(0); + }); + + test('a page added after the first run is picked up (progression, not a frozen set)', async () => { + const body = 'Another opinion-bearing body long enough to clear the eligibility floor. '.repeat(5); + await engine.putPage('concepts/progression-c', { + type: 'concept', title: 'C', compiled_truth: body, frontmatter: {}, + }); + const r3 = await extractTakesFromPages(engine, { bootstrapEnabled: true, maxPages: 50 }); + expect(r3.pages_scanned).toBe(1); + }); + + test('includeCovered rescans everything (refresh semantics)', async () => { + const r4 = await extractTakesFromPages(engine, { + bootstrapEnabled: true, maxPages: 50, includeCovered: true, + }); + expect(r4.pages_scanned).toBe(3); + }); +});