mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
The Dream cycle disables sync's inline extraction and routes changed slugs through extractForSlugs, which flushed link/timeline batches but never stamped links_extracted_at — so incrementally extracted pages stayed permanently visible to `extract --stale` / doctor. Collect processedRefs per successfully processed page and stamp them via stampExtracted (best-effort) after both batch flushes, non-dry-run mode 'all' only. Source-id threading from the original PR #2637 already landed on master via #1503/#1747, so this rebase carries only the missing watermark stamp plus regression tests. Takeover of #2637. Co-authored-by: Garry Tan <garrytan@gmail.com> Co-authored-by: JavanC <JavanC@users.noreply.github.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Garry Tan
JavanC
Claude Fable 5
parent
58606cc924
commit
8dc3310483
@@ -1025,6 +1025,10 @@ async function extractForSlugs(
|
||||
let linksCreated = 0;
|
||||
let timelineCreated = 0;
|
||||
let pagesProcessed = 0;
|
||||
// #2636: successfully processed pages get their extraction watermark
|
||||
// stamped after the final flush (mode 'all' only — a partial-mode run
|
||||
// hasn't done the full extraction the watermark asserts).
|
||||
const processedRefs: Array<{ slug: string; source_id: string }> = [];
|
||||
|
||||
// Issue #972: read the basename flag once per extract run.
|
||||
const globalBasename = await isGlobalBasenameEnabled(engine);
|
||||
@@ -1113,6 +1117,7 @@ async function extractForSlugs(
|
||||
}
|
||||
|
||||
pagesProcessed++;
|
||||
if (!dryRun) processedRefs.push({ slug, source_id: sourceId ?? 'default' });
|
||||
} catch { /* skip unreadable */ }
|
||||
progress.tick(1);
|
||||
},
|
||||
@@ -1120,6 +1125,13 @@ async function extractForSlugs(
|
||||
|
||||
await flushLinks();
|
||||
await flushTimeline();
|
||||
// #2636: the Dream cycle disables sync's inline extraction and routes
|
||||
// changed slugs through this incremental path — without a stamp here,
|
||||
// those pages never get links_extracted_at and stay permanently visible
|
||||
// to `extract --stale` / doctor. Stamp only after BOTH batches flushed.
|
||||
if (!dryRun && mode === 'all') {
|
||||
await stampExtracted(engine, processedRefs);
|
||||
}
|
||||
progress.finish();
|
||||
|
||||
if (!jsonMode) {
|
||||
|
||||
@@ -60,6 +60,51 @@ async function seedPage(slug: string, body: string): Promise<void> {
|
||||
}
|
||||
|
||||
describe('runExtractCore — incremental cycle path (#417)', () => {
|
||||
test('Dream incremental all-mode stamps the source-scoped extraction watermark (#2636)', async () => {
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO sources (id, name, local_path) VALUES ($1, $2, $3)`,
|
||||
['repo-a', 'repo-a', tempDir],
|
||||
);
|
||||
await engine.putPage('people/alice-example', {
|
||||
type: 'person',
|
||||
title: 'alice-example',
|
||||
compiled_truth: '# alice',
|
||||
timeline: '',
|
||||
frontmatter: {},
|
||||
content_hash: 'h',
|
||||
}, { sourceId: 'repo-a' });
|
||||
writeFileSync(join(tempDir, 'people/alice-example.md'), '# alice');
|
||||
|
||||
await runExtractCore(engine as unknown as BrainEngine, {
|
||||
mode: 'all',
|
||||
dir: tempDir,
|
||||
slugs: ['people/alice-example'],
|
||||
sourceId: 'repo-a',
|
||||
});
|
||||
|
||||
const rows = await engine.executeRaw<{ links_extracted_at: string | null }>(
|
||||
`SELECT links_extracted_at FROM pages WHERE slug = $1 AND source_id = $2`,
|
||||
['people/alice-example', 'repo-a'],
|
||||
);
|
||||
expect(rows[0]?.links_extracted_at).not.toBeNull();
|
||||
expect(await engine.countStalePagesForExtraction({ sourceId: 'repo-a' })).toBe(0);
|
||||
});
|
||||
|
||||
test('Dream incremental dry-run does NOT stamp the watermark', async () => {
|
||||
await seedPage('people/alice-example', '# alice');
|
||||
await runExtractCore(engine as unknown as BrainEngine, {
|
||||
mode: 'all',
|
||||
dir: tempDir,
|
||||
slugs: ['people/alice-example'],
|
||||
dryRun: true,
|
||||
});
|
||||
const rows = await engine.executeRaw<{ links_extracted_at: string | null }>(
|
||||
`SELECT links_extracted_at FROM pages WHERE slug = $1`,
|
||||
['people/alice-example'],
|
||||
);
|
||||
expect(rows[0]?.links_extracted_at ?? null).toBeNull();
|
||||
});
|
||||
|
||||
test('1. slugs: [] returns immediately with zero counts (early-return path)', async () => {
|
||||
await seedPage('people/alice-example', '# alice');
|
||||
const result = await runExtractCore(engine as unknown as BrainEngine, {
|
||||
|
||||
Reference in New Issue
Block a user