mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
v0.42.7.0 feat(extract): link/timeline extraction freshness watermark — gbrain extract --stale + doctor lag check (#1696) (#1755)
* feat(extract): link/timeline extraction freshness watermark (#1696) Closes the "imported != curated" gap: plain `gbrain sync` only extracts CHANGED pages, so a brain with autopilot off accumulated a links table that was ~99.7% untyped `mentions` with nothing surfacing it. Adds a per-page freshness watermark (pages.links_extracted_at, migration v112) and three things built on it: - `gbrain extract --stale [--source-id] [--catch-up] [--dry-run] [--json]`: incremental DB-source link+timeline sweep over pages whose extraction is stale (never extracted, edited since, or extractor version bumped). Small byte-bounded batches, non-swallowing flush, stamp-after-flush so a crash re-extracts idempotently. Stamps with the row's READ updated_at (not now()) so a concurrent edit during the sweep stays stale instead of being lost. - `links_extraction_lag` doctor check (local + remote): warn-only by default (>20%), hard-fail only via GBRAIN_EXTRACTION_LAG_FAIL_PCT. Vacuous-skip <100 pages; pre-v112 brains graceful-skip. - `gbrain sync --no-extract` flag + end-of-sync nudge (fires on synced|first_sync|up_to_date so the initial import surfaces its backlog). Three new BrainEngine methods (countStalePagesForExtraction / listStalePagesForExtraction / markPagesExtractedBatch) with Postgres<->PGLite parity + bootstrap probes. Schema parity: schema.sql + regenerated pglite-schema.ts + schema-embedded.ts + bootstrap-coverage test. Migration v112 (composite (source_id, links_extracted_at) index, no backfill so the real backlog surfaces on first doctor run). * test(audit): hermetic GBRAIN_AUDIT_DIR override for prune ENOENT case The "no-op when audit dir does not exist (ENOENT)" case called pruneOldBatchRetryAuditFiles without a GBRAIN_AUDIT_DIR override, so it read the developer's real ~/.gbrain/audit and flaked (kept>0) on any machine with prior gbrain audit history. Point it at a guaranteed-nonexistent temp path so it tests the real missing-dir branch hermetically — matching the file header's "never touches ~/.gbrain/audit" contract. Pre-existing flake (introduced by v0.41.19.0 #1537), unrelated to #1696. * chore: bump version and changelog (v0.42.2.0) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * docs: CLAUDE.md key-files entry for the #1696 extract-stale wave + regen llms-full --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
662a6e27d4
commit
ca68a551db
@@ -344,6 +344,7 @@ describe('BATCH_AUDIT_SITES typed enum + isBatchAuditSite guard (D10c codex)', (
|
||||
'extract.links_fs', 'extract.timeline_fs',
|
||||
'extract.links_db', 'extract.timeline_db',
|
||||
'extract.by_mention',
|
||||
'extract.stale',
|
||||
'mcp.put_page.autolink',
|
||||
'sync.import_file',
|
||||
'reindex.markdown', 'reindex.multimodal',
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
/**
|
||||
* Tests for the `links_extraction_lag` doctor check (v0.42.7, #1696).
|
||||
* Hermetic PGLite. Bulk-seeds pages via raw SQL (the check only does COUNT +
|
||||
* countStalePagesForExtraction — no real ingestion needed).
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { resetPgliteState } from './helpers/reset-pglite.ts';
|
||||
import { withEnv } from './helpers/with-env.ts';
|
||||
import { checkLinksExtractionLag } from '../src/commands/doctor.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
if (engine) await engine.disconnect();
|
||||
}, 60_000);
|
||||
|
||||
beforeEach(async () => {
|
||||
await resetPgliteState(engine);
|
||||
});
|
||||
|
||||
/** Bulk-insert N pages under a source; all start with links_extracted_at NULL. */
|
||||
async function seedPages(n: number, sourceId = 'default', prefix = 'p'): Promise<void> {
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO pages (slug, source_id, type, title, compiled_truth, timeline, frontmatter, content_hash, created_at, updated_at)
|
||||
SELECT $1 || '/' || g, $2, 'concept', 'T' || g, '', '', '{}'::jsonb, $1 || 'h' || g, now(), now()
|
||||
FROM generate_series(1, $3) g`,
|
||||
[prefix, sourceId, n],
|
||||
);
|
||||
}
|
||||
|
||||
describe('links_extraction_lag doctor check', () => {
|
||||
test('no pages → ok (not applicable)', async () => {
|
||||
const c = await checkLinksExtractionLag(engine);
|
||||
expect(c.status).toBe('ok');
|
||||
expect(c.message).toContain('no pages');
|
||||
});
|
||||
|
||||
test('<100 pages, no --source → ok (vacuous-skip)', async () => {
|
||||
await seedPages(50);
|
||||
const c = await checkLinksExtractionLag(engine);
|
||||
expect(c.status).toBe('ok');
|
||||
expect(c.message).toContain('too few');
|
||||
});
|
||||
|
||||
test('>100 pages, all un-extracted → warn (>20%)', async () => {
|
||||
await seedPages(120);
|
||||
const c = await checkLinksExtractionLag(engine);
|
||||
expect(c.status).toBe('warn');
|
||||
expect(c.message).toContain('un-extracted edges');
|
||||
expect(c.message).toContain('gbrain extract --stale');
|
||||
expect((c.details as any).pct).toBe(100);
|
||||
});
|
||||
|
||||
test('>100 pages, all stamped fresh → ok', async () => {
|
||||
await seedPages(120);
|
||||
await engine.executeRaw(`UPDATE pages SET links_extracted_at = now()`);
|
||||
const c = await checkLinksExtractionLag(engine);
|
||||
expect(c.status).toBe('ok');
|
||||
expect(c.message).toContain('Extraction current');
|
||||
});
|
||||
|
||||
test('warn-only by default: 100% stale does NOT fail without fail-pct', async () => {
|
||||
await seedPages(120);
|
||||
const c = await checkLinksExtractionLag(engine);
|
||||
expect(c.status).toBe('warn'); // never 'fail' by default
|
||||
});
|
||||
|
||||
test('GBRAIN_EXTRACTION_LAG_FAIL_PCT opts into hard fail', async () => {
|
||||
await seedPages(120);
|
||||
await withEnv({ GBRAIN_EXTRACTION_LAG_FAIL_PCT: '50' }, async () => {
|
||||
const c = await checkLinksExtractionLag(engine);
|
||||
expect(c.status).toBe('fail');
|
||||
expect(c.message).toContain('fail threshold');
|
||||
});
|
||||
});
|
||||
|
||||
test('GBRAIN_EXTRACTION_LAG_WARN_PCT raises the warn bar', async () => {
|
||||
// 10 of 120 stale = ~8%. Default warn 20% → ok. Lower to 5% → warn.
|
||||
await seedPages(120);
|
||||
await engine.executeRaw(`UPDATE pages SET links_extracted_at = now() WHERE slug NOT IN (SELECT slug FROM pages ORDER BY id LIMIT 10)`);
|
||||
const ok = await checkLinksExtractionLag(engine);
|
||||
expect(ok.status).toBe('ok');
|
||||
await withEnv({ GBRAIN_EXTRACTION_LAG_WARN_PCT: '5' }, async () => {
|
||||
const warn = await checkLinksExtractionLag(engine);
|
||||
expect(warn.status).toBe('warn');
|
||||
});
|
||||
});
|
||||
|
||||
test('--source scope: small source IS assessed (no vacuous-skip)', async () => {
|
||||
// 10 pages under source 'dept-x' — below the 100 floor, but explicit
|
||||
// --source means we assess it anyway (mirrors orphan_ratio).
|
||||
await engine.executeRaw(`INSERT INTO sources (id, name, config) VALUES ('dept-x', 'Dept X', '{}'::jsonb) ON CONFLICT DO NOTHING`);
|
||||
await seedPages(10, 'dept-x', 'dx');
|
||||
const c = await checkLinksExtractionLag(engine, { sourceId: 'dept-x' });
|
||||
expect(c.status).toBe('warn'); // all 10 stale, assessed despite < 100
|
||||
expect(c.message).toContain("source 'dept-x'");
|
||||
});
|
||||
|
||||
test('pre-v112 brain (column missing) → ok (graceful)', async () => {
|
||||
await seedPages(120);
|
||||
// Simulate a pre-v112 brain by dropping the column.
|
||||
await engine.executeRaw(`ALTER TABLE pages DROP COLUMN links_extracted_at`);
|
||||
try {
|
||||
const c = await checkLinksExtractionLag(engine);
|
||||
expect(c.status).toBe('ok');
|
||||
expect(c.message).toContain('pre-v112');
|
||||
} finally {
|
||||
// Restore so resetPgliteState's TRUNCATE-only reset leaves a valid schema
|
||||
// for the next test (the column is re-added; data is wiped by beforeEach).
|
||||
await engine.executeRaw(`ALTER TABLE pages ADD COLUMN links_extracted_at TIMESTAMPTZ`);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -413,6 +413,66 @@ describeBoth('Engine parity — Postgres vs PGLite', () => {
|
||||
expect(pgliteFed).toEqual(pgFed);
|
||||
});
|
||||
|
||||
// v0.42.7 (#1696): stale-page extraction watermark parity. Isolated under a
|
||||
// dedicated source so other tests' mutations don't perturb the counts.
|
||||
test('stale-page extraction methods: Postgres ↔ PGLite parity', async () => {
|
||||
const SRC = 'stale-parity';
|
||||
const VER = '2026-05-31T00:00:00Z';
|
||||
for (const eng of [pgEngine, pgliteEngine]) {
|
||||
await eng.executeRaw(`INSERT INTO sources (id, name, config) VALUES ($1, 'Stale Parity', '{}'::jsonb) ON CONFLICT DO NOTHING`, [SRC]);
|
||||
await eng.executeRaw(
|
||||
`INSERT INTO pages (slug, source_id, type, title, compiled_truth, timeline, frontmatter, content_hash, created_at, updated_at)
|
||||
SELECT 'sp/' || g, $1, 'concept', 'SP' || g, 'body ' || g, '', '{}'::jsonb, 'sph' || g, now(), now()
|
||||
FROM generate_series(1, 3) g`,
|
||||
[SRC],
|
||||
);
|
||||
}
|
||||
|
||||
// NULL arm: all 3 stale on both engines.
|
||||
expect(await pgEngine.countStalePagesForExtraction({ sourceId: SRC })).toBe(3);
|
||||
expect(await pgliteEngine.countStalePagesForExtraction({ sourceId: SRC })).toBe(3);
|
||||
|
||||
// listStalePagesForExtraction: same slugs + content columns populated.
|
||||
const pgList = (await pgEngine.listStalePagesForExtraction({ batchSize: 10, sourceId: SRC })).map(r => r.slug).sort();
|
||||
const plList = (await pgliteEngine.listStalePagesForExtraction({ batchSize: 10, sourceId: SRC })).map(r => r.slug).sort();
|
||||
expect(pgList).toEqual(['sp/1', 'sp/2', 'sp/3']);
|
||||
expect(plList).toEqual(pgList);
|
||||
const pgRow = (await pgEngine.listStalePagesForExtraction({ batchSize: 1, sourceId: SRC }))[0];
|
||||
expect(pgRow.compiled_truth).toBeTruthy();
|
||||
expect(pgRow.updated_at).toBeInstanceOf(Date);
|
||||
|
||||
// markPagesExtractedBatch: stamp one → count drops to 2 on both.
|
||||
const stampAt = new Date().toISOString();
|
||||
await pgEngine.markPagesExtractedBatch([{ slug: 'sp/1', source_id: SRC }], stampAt);
|
||||
await pgliteEngine.markPagesExtractedBatch([{ slug: 'sp/1', source_id: SRC }], stampAt);
|
||||
expect(await pgEngine.countStalePagesForExtraction({ sourceId: SRC })).toBe(2);
|
||||
expect(await pgliteEngine.countStalePagesForExtraction({ sourceId: SRC })).toBe(2);
|
||||
|
||||
// version arm: stamp sp/2 old + set updated_at old (isolate version arm) →
|
||||
// flagged only when versionTs is passed. Parity on both engines.
|
||||
for (const eng of [pgEngine, pgliteEngine]) {
|
||||
await eng.markPagesExtractedBatch([{ slug: 'sp/2', source_id: SRC }], '2000-01-01T00:00:00Z');
|
||||
await eng.executeRaw(`UPDATE pages SET updated_at = '2000-01-01T00:00:00Z' WHERE slug = 'sp/2' AND source_id = $1`, [SRC]);
|
||||
}
|
||||
// Without versionTs: sp/2 not stale (stamp == updated, not NULL). sp/3 still NULL-stale.
|
||||
expect(await pgEngine.countStalePagesForExtraction({ sourceId: SRC })).toBe(1);
|
||||
expect(await pgliteEngine.countStalePagesForExtraction({ sourceId: SRC })).toBe(1);
|
||||
// With versionTs: sp/2's old stamp (< VER) re-flags it → 2 stale.
|
||||
expect(await pgEngine.countStalePagesForExtraction({ sourceId: SRC, versionTs: VER })).toBe(2);
|
||||
expect(await pgliteEngine.countStalePagesForExtraction({ sourceId: SRC, versionTs: VER })).toBe(2);
|
||||
|
||||
// edited-since arm: stamp sp/1 in the recent past, updated_at slightly after →
|
||||
// re-flagged on both engines (updated_at > links_extracted_at).
|
||||
for (const eng of [pgEngine, pgliteEngine]) {
|
||||
await eng.executeRaw(
|
||||
`UPDATE pages SET links_extracted_at = now() - interval '2 hours', updated_at = now() - interval '1 hour' WHERE slug = 'sp/1' AND source_id = $1`,
|
||||
[SRC],
|
||||
);
|
||||
}
|
||||
expect(await pgEngine.countStalePagesForExtraction({ sourceId: SRC })).toBe(2); // sp/1 (edited) + sp/3 (NULL)
|
||||
expect(await pgliteEngine.countStalePagesForExtraction({ sourceId: SRC })).toBe(2);
|
||||
});
|
||||
|
||||
test('v0.41.39 listEnrichCandidates parity (thin filter + source-aware inbound + order)', async () => {
|
||||
const stub = 'Stub page.';
|
||||
const pageSql = `
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
/**
|
||||
* Tests for `gbrain extract --stale` + the link-extraction freshness watermark
|
||||
* (v0.42.7, #1696). Hermetic PGLite — no DATABASE_URL, no API keys.
|
||||
*
|
||||
* Covers:
|
||||
* - engine methods: countStalePagesForExtraction (NULL / version / edited-since
|
||||
* arms + source scope), listStalePagesForExtraction (content + keyset),
|
||||
* markPagesExtractedBatch (composite-key stamp).
|
||||
* - `extract --stale`: sweep creates typed edges + stamps every processed page
|
||||
* (incl. zero-link), second run finds 0 stale (idempotent), --dry-run writes
|
||||
* nothing, --source-id scope.
|
||||
* - CRITICAL regression (CDX-1): a page edited after a prior stamp
|
||||
* (updated_at > links_extracted_at) is re-flagged stale and re-extracted.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { runExtract } from '../src/commands/extract.ts';
|
||||
import { LINK_EXTRACTOR_VERSION_TS } from '../src/core/link-extraction.ts';
|
||||
import type { PageInput } from '../src/core/types.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
if (engine) await engine.disconnect();
|
||||
}, 60_000);
|
||||
|
||||
async function truncateAll() {
|
||||
for (const t of ['content_chunks', 'links', 'tags', 'raw_data', 'timeline_entries', 'page_versions', 'ingest_log', 'pages']) {
|
||||
await (engine as any).db.exec(`DELETE FROM ${t}`);
|
||||
}
|
||||
}
|
||||
beforeEach(truncateAll);
|
||||
|
||||
const personPage = (title: string, body = ''): PageInput => ({ type: 'person', title, compiled_truth: body, timeline: '' });
|
||||
const companyPage = (title: string, body = ''): PageInput => ({ type: 'company', title, compiled_truth: body, timeline: '' });
|
||||
|
||||
async function stampOf(slug: string, sourceId = 'default'): Promise<string | null> {
|
||||
const rows = await engine.executeRaw<{ links_extracted_at: string | null }>(
|
||||
`SELECT links_extracted_at FROM pages WHERE slug = $1 AND source_id = $2`, [slug, sourceId],
|
||||
);
|
||||
return rows[0]?.links_extracted_at ?? null;
|
||||
}
|
||||
|
||||
describe('engine: stale-page extraction methods', () => {
|
||||
test('countStalePagesForExtraction: NULL arm counts never-extracted pages', async () => {
|
||||
await engine.putPage('people/alice', personPage('Alice'));
|
||||
await engine.putPage('people/bob', personPage('Bob'));
|
||||
expect(await engine.countStalePagesForExtraction()).toBe(2);
|
||||
});
|
||||
|
||||
test('countStalePagesForExtraction: stamped pages drop out', async () => {
|
||||
await engine.putPage('people/alice', personPage('Alice'));
|
||||
await engine.markPagesExtractedBatch([{ slug: 'people/alice', source_id: 'default' }], new Date().toISOString());
|
||||
expect(await engine.countStalePagesForExtraction()).toBe(0);
|
||||
});
|
||||
|
||||
test('countStalePagesForExtraction: version arm flags pre-version stamps', async () => {
|
||||
await engine.putPage('people/alice', personPage('Alice'));
|
||||
// Stamp with an OLD timestamp (before LINK_EXTRACTOR_VERSION_TS).
|
||||
await engine.markPagesExtractedBatch([{ slug: 'people/alice', source_id: 'default' }], '2000-01-01T00:00:00Z');
|
||||
// Without versionTs: only NULL/edited arms → not stale (stamp >= updated? no:
|
||||
// stamp is 2000, updated is now → updated_at > stamp → STALE via edited arm).
|
||||
// So set updated_at back too, isolating the version arm:
|
||||
await engine.executeRaw(`UPDATE pages SET updated_at = '2000-01-01T00:00:00Z' WHERE slug = 'people/alice'`);
|
||||
expect(await engine.countStalePagesForExtraction()).toBe(0); // no version, stamp==updated, not NULL
|
||||
expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(1); // version arm
|
||||
});
|
||||
|
||||
test('countStalePagesForExtraction: edited-since arm (CDX-1)', async () => {
|
||||
await engine.putPage('people/alice', personPage('Alice'));
|
||||
await engine.markPagesExtractedBatch([{ slug: 'people/alice', source_id: 'default' }], new Date().toISOString());
|
||||
expect(await engine.countStalePagesForExtraction()).toBe(0);
|
||||
// Simulate an edit AFTER the stamp (put_page / sync --no-extract).
|
||||
await engine.executeRaw(`UPDATE pages SET updated_at = '2099-01-01T00:00:00Z' WHERE slug = 'people/alice'`);
|
||||
expect(await engine.countStalePagesForExtraction()).toBe(1);
|
||||
});
|
||||
|
||||
test('listStalePagesForExtraction: returns content columns + keyset paginates', async () => {
|
||||
await engine.putPage('people/alice', personPage('Alice', 'Body A'));
|
||||
await engine.putPage('people/bob', personPage('Bob', 'Body B'));
|
||||
const batch1 = await engine.listStalePagesForExtraction({ batchSize: 1 });
|
||||
expect(batch1.length).toBe(1);
|
||||
expect(batch1[0].compiled_truth).toBeTruthy();
|
||||
expect(batch1[0].title).toBeTruthy();
|
||||
expect(batch1[0].frontmatter).toBeDefined();
|
||||
const batch2 = await engine.listStalePagesForExtraction({ batchSize: 10, afterPageId: batch1[0].id });
|
||||
expect(batch2.length).toBe(1);
|
||||
expect(batch2[0].id).toBeGreaterThan(batch1[0].id);
|
||||
});
|
||||
|
||||
test('markPagesExtractedBatch: empty input is a no-op', async () => {
|
||||
await engine.markPagesExtractedBatch([], new Date().toISOString());
|
||||
expect(true).toBe(true); // no throw
|
||||
});
|
||||
});
|
||||
|
||||
describe('gbrain extract --stale', () => {
|
||||
test('extracts typed edges + stamps every processed page (incl. zero-link)', async () => {
|
||||
await engine.putPage('people/alice', personPage('Alice'));
|
||||
await engine.putPage('companies/acme', companyPage('Acme', '[Alice](people/alice) is the CEO of [Acme](companies/acme).'));
|
||||
await engine.putPage('people/lonely', personPage('Lonely', 'No links here.'));
|
||||
|
||||
await runExtract(engine, ['--stale']);
|
||||
|
||||
const links = await engine.getLinks('companies/acme');
|
||||
expect(links.some(l => l.to_slug === 'people/alice')).toBe(true);
|
||||
// EVERY processed page stamped — including the zero-link one.
|
||||
expect(await stampOf('people/alice')).not.toBeNull();
|
||||
expect(await stampOf('companies/acme')).not.toBeNull();
|
||||
expect(await stampOf('people/lonely')).not.toBeNull();
|
||||
// Nothing left stale.
|
||||
expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(0);
|
||||
});
|
||||
|
||||
test('idempotent: second run finds 0 stale and creates no new links', async () => {
|
||||
await engine.putPage('people/alice', personPage('Alice'));
|
||||
await engine.putPage('companies/acme', companyPage('Acme', '[Alice](people/alice) advises [Acme](companies/acme).'));
|
||||
|
||||
await runExtract(engine, ['--stale']);
|
||||
const after1 = (await engine.getLinks('companies/acme')).length;
|
||||
const stamp1 = await stampOf('companies/acme');
|
||||
|
||||
await runExtract(engine, ['--stale']);
|
||||
const after2 = (await engine.getLinks('companies/acme')).length;
|
||||
expect(after2).toBe(after1);
|
||||
// Second run had 0 stale → did not re-stamp (stamp unchanged is acceptable;
|
||||
// the key invariant is no duplicate links).
|
||||
expect(stamp1).not.toBeNull();
|
||||
});
|
||||
|
||||
test('--dry-run reports count and writes nothing', async () => {
|
||||
await engine.putPage('people/alice', personPage('Alice'));
|
||||
await engine.putPage('companies/acme', companyPage('Acme', '[Alice](people/alice) joined [Acme](companies/acme).'));
|
||||
|
||||
await runExtract(engine, ['--stale', '--dry-run']);
|
||||
|
||||
expect(await engine.getLinks('companies/acme')).toHaveLength(0);
|
||||
expect(await stampOf('people/alice')).toBeNull();
|
||||
expect(await stampOf('companies/acme')).toBeNull();
|
||||
// Still stale after dry-run.
|
||||
expect(await engine.countStalePagesForExtraction()).toBe(2);
|
||||
});
|
||||
|
||||
test('CRITICAL (CDX-1): page edited after stamp is re-extracted', async () => {
|
||||
await engine.putPage('people/alice', personPage('Alice'));
|
||||
await engine.putPage('companies/acme', companyPage('Acme', 'No links yet.'));
|
||||
await runExtract(engine, ['--stale']);
|
||||
expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(0);
|
||||
|
||||
// Simulate an edit that adds a link WITHOUT extracting (MCP put_page /
|
||||
// sync --no-extract). Use relative intervals so it's clock-agnostic: the
|
||||
// stamp + edit both land in the recent past (after LINK_EXTRACTOR_VERSION_TS),
|
||||
// with updated_at AFTER the stamp — and crucially both BEFORE real-now, so
|
||||
// the re-extract's now()-stamp deterministically supersedes the edit.
|
||||
await engine.executeRaw(
|
||||
`UPDATE pages
|
||||
SET compiled_truth = $1,
|
||||
links_extracted_at = now() - interval '2 hours',
|
||||
updated_at = now() - interval '1 hour'
|
||||
WHERE slug = 'companies/acme'`,
|
||||
['[Alice](people/alice) now works at [Acme](companies/acme).'],
|
||||
);
|
||||
// Re-flagged stale by the updated_at arm (updated > stamp).
|
||||
expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(1);
|
||||
|
||||
// extract --stale picks it up, creates the now-present edge, and re-stamps
|
||||
// at now() (> the edit's updated_at) so the page is fresh again.
|
||||
await runExtract(engine, ['--stale']);
|
||||
const links = await engine.getLinks('companies/acme');
|
||||
expect(links.some(l => l.to_slug === 'people/alice')).toBe(true);
|
||||
expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(0);
|
||||
});
|
||||
|
||||
test('CDX-4 (D2): a link-flush throw aborts the sweep and leaves pages UNSTAMPED', async () => {
|
||||
await engine.putPage('people/alice', personPage('Alice'));
|
||||
await engine.putPage('companies/acme', companyPage('Acme', '[Alice](people/alice) founded [Acme](companies/acme).'));
|
||||
|
||||
// Make the link flush throw mid-sweep. The --stale path flushes
|
||||
// NON-swallowing (no try/catch), so the throw must propagate AND no page in
|
||||
// the batch may be stamped (stamp runs only AFTER a successful flush).
|
||||
const origBatch = engine.addLinksBatch.bind(engine);
|
||||
let threw = false;
|
||||
(engine as unknown as { addLinksBatch: unknown }).addLinksBatch = async () => { throw new Error('__flush_boom__'); };
|
||||
try {
|
||||
await runExtract(engine, ['--stale']);
|
||||
} catch (e) {
|
||||
if ((e as Error).message === '__flush_boom__') threw = true; else throw e;
|
||||
} finally {
|
||||
(engine as unknown as { addLinksBatch: unknown }).addLinksBatch = origBatch;
|
||||
}
|
||||
expect(threw).toBe(true);
|
||||
// Pages whose edges were lost are NOT stamped fresh — they stay stale.
|
||||
expect(await stampOf('people/alice')).toBeNull();
|
||||
expect(await stampOf('companies/acme')).toBeNull();
|
||||
expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(2);
|
||||
|
||||
// A clean re-run re-extracts idempotently (ON CONFLICT DO NOTHING).
|
||||
await runExtract(engine, ['--stale']);
|
||||
expect((await engine.getLinks('companies/acme')).some(l => l.to_slug === 'people/alice')).toBe(true);
|
||||
expect(await stampOf('companies/acme')).not.toBeNull();
|
||||
expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(0);
|
||||
});
|
||||
|
||||
test('D4 race: a concurrent edit landing during the sweep is NOT masked', async () => {
|
||||
await engine.putPage('people/alice', personPage('Alice'));
|
||||
await engine.putPage('companies/acme', companyPage('Acme', '[Alice](people/alice) backs [Acme](companies/acme).'));
|
||||
// Anchor acme's updated_at in the past so the read value is well-defined.
|
||||
await engine.executeRaw(`UPDATE pages SET updated_at = now() - interval '3 hours' WHERE slug = 'companies/acme'`);
|
||||
|
||||
// Simulate an edit landing BETWEEN the list read (updated_at = now-3h) and
|
||||
// the stamp: bump acme's updated_at to now-1h just before the real stamp.
|
||||
// D4 stamps with the READ updated_at (now-3h), so now-1h > now-3h → acme
|
||||
// stays stale (edit preserved). The OLD now()-stamp would set
|
||||
// links_extracted_at = now > now-1h → acme marked fresh, edit silently lost.
|
||||
const origStamp = engine.markPagesExtractedBatch.bind(engine);
|
||||
let hooked = false;
|
||||
(engine as unknown as { markPagesExtractedBatch: unknown }).markPagesExtractedBatch = async (
|
||||
refs: Array<{ slug: string; source_id: string; extractedAt?: string }>, def: string,
|
||||
) => {
|
||||
if (!hooked) {
|
||||
hooked = true;
|
||||
await engine.executeRaw(`UPDATE pages SET updated_at = now() - interval '1 hour' WHERE slug = 'companies/acme'`);
|
||||
}
|
||||
return origStamp(refs, def);
|
||||
};
|
||||
try {
|
||||
await runExtract(engine, ['--stale']);
|
||||
} finally {
|
||||
(engine as unknown as { markPagesExtractedBatch: unknown }).markPagesExtractedBatch = origStamp;
|
||||
}
|
||||
expect(hooked).toBe(true);
|
||||
// acme stays stale (only the concurrently-edited page); alice was stamped
|
||||
// with its own read updated_at and is fresh.
|
||||
expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(1);
|
||||
});
|
||||
|
||||
test('--source fs is rejected (DB-source only)', async () => {
|
||||
const origErr = console.error;
|
||||
const origExit = process.exit;
|
||||
let exited = false; let msg = '';
|
||||
console.error = (m?: unknown) => { msg += String(m); };
|
||||
process.exit = ((_code?: number) => { exited = true; throw new Error('__exit__'); }) as unknown as typeof process.exit;
|
||||
try {
|
||||
await runExtract(engine, ['--stale', '--source', 'fs']);
|
||||
} catch (e) {
|
||||
if ((e as Error).message !== '__exit__') throw e;
|
||||
} finally {
|
||||
console.error = origErr;
|
||||
process.exit = origExit;
|
||||
}
|
||||
expect(exited).toBe(true);
|
||||
expect(msg).toContain('DB-source only');
|
||||
});
|
||||
});
|
||||
@@ -2139,3 +2139,43 @@ describe('migrate v89 — round-trip on PGLite', () => {
|
||||
});
|
||||
});
|
||||
|
||||
// v0.42.7 (#1696): pages_links_extracted_at watermark migration.
|
||||
describe('v112 — pages_links_extracted_at', () => {
|
||||
let engine: PGLiteEngine;
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
}, 60_000);
|
||||
afterAll(async () => { if (engine) await engine.disconnect(); }, 60_000);
|
||||
|
||||
test('v112 entry exists with the documented name + transaction:false + handler', () => {
|
||||
const m = MIGRATIONS.find(x => x.version === 112);
|
||||
expect(m).toBeDefined();
|
||||
expect(m!.name).toBe('pages_links_extracted_at');
|
||||
expect(m!.transaction).toBe(false);
|
||||
expect(typeof m!.handler).toBe('function');
|
||||
});
|
||||
|
||||
test('LATEST_VERSION is at or above 112', () => {
|
||||
expect(LATEST_VERSION).toBeGreaterThanOrEqual(112);
|
||||
});
|
||||
|
||||
test('links_extracted_at column exists after initSchema, nullable, TIMESTAMPTZ', async () => {
|
||||
const rows = await engine.executeRaw<{ is_nullable: string; data_type: string }>(
|
||||
`SELECT is_nullable, data_type FROM information_schema.columns
|
||||
WHERE table_name = 'pages' AND column_name = 'links_extracted_at'`, [],
|
||||
);
|
||||
expect(rows.length).toBe(1);
|
||||
expect(rows[0].is_nullable).toBe('YES');
|
||||
expect(rows[0].data_type.toLowerCase()).toContain('timestamp');
|
||||
});
|
||||
|
||||
test('composite index pages_links_extracted_at_idx exists after initSchema', async () => {
|
||||
const rows = await engine.executeRaw<{ indexname: string }>(
|
||||
`SELECT indexname FROM pg_indexes WHERE tablename = 'pages' AND indexname = 'pages_links_extracted_at_idx'`, [],
|
||||
);
|
||||
expect(rows.length).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -162,6 +162,12 @@ const REQUIRED_BOOTSTRAP_COVERAGE: ForwardReference[] = [
|
||||
// semantics. No SCHEMA_SQL index references it; bootstrap probe is
|
||||
// defense-in-depth (and satisfies the MIGRATIONS ADD COLUMN coverage gate).
|
||||
{ kind: 'column', table: 'pages', column: 'embedding_signature' },
|
||||
// v0.42.7 (v112) — forward-referenced by `CREATE INDEX
|
||||
// pages_links_extracted_at_idx ON pages (source_id, links_extracted_at)`.
|
||||
// Pre-v112 brains have pages without this column; bootstrap adds it before
|
||||
// SCHEMA_SQL replay creates the index. Powers `gbrain extract --stale` + the
|
||||
// `links_extraction_lag` doctor check.
|
||||
{ kind: 'column', table: 'pages', column: 'links_extracted_at' },
|
||||
];
|
||||
|
||||
test('applyForwardReferenceBootstrap covers every forward reference declared in REQUIRED_BOOTSTRAP_COVERAGE', async () => {
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
/**
|
||||
* CRITICAL regression (v0.42.7, #1696, CDX-6) — inline sync extract stamps the
|
||||
* link-extraction watermark.
|
||||
*
|
||||
* `performSync`'s INCREMENTAL path runs link/timeline extraction inline for the
|
||||
* changed pages (the `gbrain sync` default — see performSyncInner's auto-extract
|
||||
* block). v0.42.7 adds a `stampExtracted` call at that call site (after
|
||||
* extractLinksForSlugs/extractTimelineForSlugs) so a normal incremental sync
|
||||
* marks the pages it just extracted as fresh — otherwise every synced page
|
||||
* would show as stale forever in the links_extraction_lag doctor check.
|
||||
*
|
||||
* NOTE: a FULL / first sync routes to performFullSync which does NOT extract
|
||||
* inline (pre-existing behavior — exactly the "imported ≠ curated" gap that
|
||||
* `extract --stale` closes). So this regression test drives the INCREMENTAL
|
||||
* path: full sync to seed, then edit + incremental sync.
|
||||
*
|
||||
* IRON RULE: pins (a) an incremental sync NOW stamps links_extracted_at for the
|
||||
* pages it processed, and (b) the existing link extraction is unchanged. Plus
|
||||
* --no-extract: the changed page stays unstamped AND no links are created.
|
||||
*
|
||||
* Marked .serial.test.ts — spawns git subprocesses + shares one PGLite engine.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach } from 'bun:test';
|
||||
import { mkdtempSync, writeFileSync, rmSync, mkdirSync } from 'fs';
|
||||
import { execSync } from 'child_process';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { resetPgliteState } from './helpers/reset-pglite.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
let repoPath: string;
|
||||
|
||||
function git(cmd: string): void { execSync(cmd, { cwd: repoPath, stdio: 'pipe' }); }
|
||||
|
||||
function writeAcme(body: string): void {
|
||||
writeFileSync(join(repoPath, 'companies/acme.md'), [
|
||||
'---', 'type: company', 'title: Acme', '---', '', body,
|
||||
].join('\n'));
|
||||
}
|
||||
|
||||
async function stampOf(slug: string): Promise<string | null> {
|
||||
const rows = await engine.executeRaw<{ links_extracted_at: string | null }>(
|
||||
`SELECT links_extracted_at FROM pages WHERE slug = $1 AND source_id = 'default'`, [slug],
|
||||
);
|
||||
return rows[0]?.links_extracted_at ?? null;
|
||||
}
|
||||
|
||||
describe('#1696 — inline sync extract stamps links_extracted_at', () => {
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
if (engine) await engine.disconnect();
|
||||
}, 60_000);
|
||||
|
||||
beforeEach(async () => {
|
||||
await resetPgliteState(engine);
|
||||
repoPath = mkdtempSync(join(tmpdir(), 'gbrain-stamp-'));
|
||||
execSync('git init', { cwd: repoPath, stdio: 'pipe' });
|
||||
execSync('git config user.email "t@t.com"', { cwd: repoPath, stdio: 'pipe' });
|
||||
execSync('git config user.name "T"', { cwd: repoPath, stdio: 'pipe' });
|
||||
mkdirSync(join(repoPath, 'people'), { recursive: true });
|
||||
mkdirSync(join(repoPath, 'companies'), { recursive: true });
|
||||
writeFileSync(join(repoPath, 'people/alice.md'), [
|
||||
'---', 'type: person', 'title: Alice', '---', '', 'Alice is a founder.',
|
||||
].join('\n'));
|
||||
writeAcme('Acme is a company.');
|
||||
git('git add -A && git commit -m "initial"');
|
||||
|
||||
// Seed: full first sync imports both pages (no inline extract on this path).
|
||||
const { performSync } = await import('../src/commands/sync.ts');
|
||||
await performSync(engine, { repoPath, full: true, noPull: true, noEmbed: true });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (repoPath) rmSync(repoPath, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test('(a) incremental sync stamps the watermark AND (b) still extracts the link', async () => {
|
||||
const { performSync } = await import('../src/commands/sync.ts');
|
||||
// Edit acme to add the link, commit → incremental sync processes it.
|
||||
// Disk-relative link form (how real brain files reference each other on
|
||||
// disk; the FS extractor resolves relative to the file's directory).
|
||||
writeAcme('[Alice](../people/alice.md) is the CEO of Acme.');
|
||||
git('git add -A && git commit -m "add link"');
|
||||
const result = await performSync(engine, { repoPath, noPull: true, noEmbed: true });
|
||||
expect(['synced', 'first_sync']).toContain(result.status);
|
||||
|
||||
// (b) extraction unchanged — the CEO-of link is created.
|
||||
const links = await engine.getLinks('companies/acme');
|
||||
expect(links.some(l => l.to_slug === 'people/alice')).toBe(true);
|
||||
|
||||
// (a) the changed page sync extracted is now stamped (not stale).
|
||||
expect(await stampOf('companies/acme')).not.toBeNull();
|
||||
}, 60_000);
|
||||
|
||||
test('--no-extract: changed page is NOT stamped and no links are created', async () => {
|
||||
const { performSync } = await import('../src/commands/sync.ts');
|
||||
// Disk-relative link form (how real brain files reference each other on
|
||||
// disk; the FS extractor resolves relative to the file's directory).
|
||||
writeAcme('[Alice](../people/alice.md) is the CEO of Acme.');
|
||||
git('git add -A && git commit -m "add link"');
|
||||
const result = await performSync(engine, { repoPath, noPull: true, noEmbed: true, noExtract: true });
|
||||
expect(['synced', 'first_sync']).toContain(result.status);
|
||||
|
||||
expect(await engine.getLinks('companies/acme')).toHaveLength(0);
|
||||
expect(await stampOf('companies/acme')).toBeNull();
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -0,0 +1,40 @@
|
||||
/**
|
||||
* v0.42.7 (#1696, D5) — the end-of-sync extraction-lag nudge status gate.
|
||||
*
|
||||
* Codex caught that the single-source nudge was gated `=== 'synced'`, which
|
||||
* skips `first_sync` (a fresh / --full import — the BIGGEST un-extracted
|
||||
* backlog, the exact 280K-page scenario #1696 exists for) and `up_to_date`.
|
||||
* `shouldNudgeAfterSync` is the pure predicate the call site now uses; this
|
||||
* pins its contract so the gate can't silently narrow back to `synced` only.
|
||||
*
|
||||
* Pure predicate test — no PGLite, no env mutation, no module stubbing (R1-R4 compliant).
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { shouldNudgeAfterSync } from '../src/commands/sync.ts';
|
||||
|
||||
describe('shouldNudgeAfterSync (D5 status gate)', () => {
|
||||
test('fires on first_sync — the biggest-backlog initial-import case', () => {
|
||||
expect(shouldNudgeAfterSync('first_sync')).toBe(true);
|
||||
});
|
||||
|
||||
test('fires on synced (incremental)', () => {
|
||||
expect(shouldNudgeAfterSync('synced')).toBe(true);
|
||||
});
|
||||
|
||||
test('fires on up_to_date (no-op sync over a pre-existing backlog)', () => {
|
||||
expect(shouldNudgeAfterSync('up_to_date')).toBe(true);
|
||||
});
|
||||
|
||||
test('does NOT fire on dry_run (preview, no real sync)', () => {
|
||||
expect(shouldNudgeAfterSync('dry_run')).toBe(false);
|
||||
});
|
||||
|
||||
test('does NOT fire on blocked_by_failures (inconsistent state)', () => {
|
||||
expect(shouldNudgeAfterSync('blocked_by_failures')).toBe(false);
|
||||
});
|
||||
|
||||
test('does NOT fire on partial (inconsistent state)', () => {
|
||||
expect(shouldNudgeAfterSync('partial')).toBe(false);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user