mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
fix(extract): --stale sweep runs the real resolver — basename resolution reaches stale pages (#2576) (#2717)
extractStaleFromDB still used the pre-#972 `includeFrontmatter ? resolver :
nullResolver` ternary. The synthetic resolver has no resolveBasenameMatches,
so the gate in extractPageLinks skipped the issue-#972 bare-wikilink pass
regardless of link_resolution.global_basename — the sweep stamped every page
as extracted while silently dropping its [[bare-name]] links. Same brain,
same pages: `extract --stale` created 0 links where `extract links --source
db` created 218.
- Always pass the real batch resolver; gate passes via extractPageLinks opts
({ skipFrontmatter: !includeFrontmatter, globalBasename }), mirroring
extractLinksFromDB — including the codex-[P1] sourceId scoping.
- Bump LINK_EXTRACTOR_VERSION_TS (documented protocol) so pages stamped by
the broken sweep re-flag stale and re-extract under the fixed logic.
- Regression tests: bare wikilink resolves on --stale with the flag ON;
still drops with the flag OFF (back-compat). The #1768 fixture now derives
its updated_at from LINK_EXTRACTOR_VERSION_TS instead of a hardcoded date,
so future version bumps can't silently flip its version arm.
Fixes bug 1 + bug 3 of #2576. Bug 2 (DIR_PATTERN gaps) is a separate
whitelist design call, intentionally not addressed here.
Co-authored-by: YMYD <53603073+OJ-OnJourney@users.noreply.github.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
YMYD
Claude Fable 5
parent
c27b2e4b0f
commit
9b8b829ca5
+13
-4
@@ -1684,9 +1684,17 @@ export async function extractStaleFromDB(
|
||||
// Batch mode = pg_trgm + exact only, NO per-name search fallback. The
|
||||
// resolution map sees ALL sources so qualified cross-source wikilinks resolve
|
||||
// even when --source-id scopes the stale SCAN.
|
||||
const resolver = makeResolver(engine, { mode: 'batch' });
|
||||
const nullResolver = { resolve: async () => null as string | null };
|
||||
const activeResolver = includeFrontmatter ? resolver : nullResolver;
|
||||
//
|
||||
// #2576 bug 1: ALWAYS the real resolver — extractPageLinks's opts gate which
|
||||
// pass runs (`skipFrontmatter` for the frontmatter pass, `globalBasename` for
|
||||
// the issue-#972 bare-wikilink pass). The former `includeFrontmatter ?
|
||||
// resolver : nullResolver` ternary predates #972; the synthetic resolver has
|
||||
// no `resolveBasenameMatches`, so the --stale sweep silently skipped basename
|
||||
// resolution even with `link_resolution.global_basename` enabled, stamping
|
||||
// pages as extracted with their bare wikilinks dropped. Mirrors
|
||||
// extractLinksFromDB (including the codex-[P1] `sourceId` scoping).
|
||||
const resolver = makeResolver(engine, { mode: 'batch', sourceId: sourceIdFilter });
|
||||
const globalBasename = await isGlobalBasenameEnabled(engine);
|
||||
const allRefs = await engine.listAllPageRefs();
|
||||
const allSlugs = new Set<string>();
|
||||
const slugToSources = new Map<string, string[]>();
|
||||
@@ -1718,7 +1726,8 @@ export async function extractStaleFromDB(
|
||||
for (const page of rows) {
|
||||
const fullContent = page.compiled_truth + '\n' + page.timeline;
|
||||
const extracted = await extractPageLinks(
|
||||
page.slug, fullContent, page.frontmatter, page.type, activeResolver,
|
||||
page.slug, fullContent, page.frontmatter, page.type, resolver,
|
||||
{ skipFrontmatter: !includeFrontmatter, globalBasename },
|
||||
);
|
||||
for (const c of extracted.candidates) {
|
||||
const r = resolveCandidateSources(c, page.slug, page.source_id, allSlugs, slugToSources);
|
||||
|
||||
@@ -28,7 +28,10 @@ import { ensureWellFormed } from './text-safe.ts';
|
||||
* OR updated_at > links_extracted_at`. It is an ISO-8601 string (NOT a number) —
|
||||
* the column is TIMESTAMPTZ and the predicate binds it as `::timestamptz`.
|
||||
*/
|
||||
export const LINK_EXTRACTOR_VERSION_TS = '2026-05-31T00:00:00Z';
|
||||
// 2026-07-10: bumped for the #2576 --stale nullResolver fix — sweeps before it
|
||||
// stamped pages with their bare wikilinks silently dropped; the bump re-flags
|
||||
// them so the fixed sweep re-extracts.
|
||||
export const LINK_EXTRACTOR_VERSION_TS = '2026-07-10T00:00:00Z';
|
||||
|
||||
// ─── Entity references ──────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -186,9 +186,12 @@ describe('gbrain extract --stale', () => {
|
||||
// the precision gap is deterministic regardless of the engine's now() granularity.
|
||||
await engine.putPage('people/alice', personPage('Alice'));
|
||||
await engine.putPage('companies/acme', companyPage('Acme', '[Alice](people/alice) advises [Acme](companies/acme).'));
|
||||
// Microsecond-precision updated_at, recent (after LINK_EXTRACTOR_VERSION_TS) so the
|
||||
// version arm doesn't fire — the edited arm is what must clear.
|
||||
await engine.executeRaw(`UPDATE pages SET updated_at = '2026-06-02 08:18:58.999166+00'`);
|
||||
// Microsecond-precision updated_at, derived from LINK_EXTRACTOR_VERSION_TS
|
||||
// (+2 days) so the version arm never fires regardless of future bumps —
|
||||
// the edited arm is what must clear.
|
||||
const afterVersionIso = new Date(Date.parse(LINK_EXTRACTOR_VERSION_TS) + 48 * 3600 * 1000).toISOString();
|
||||
const usUpdatedAt = `${afterVersionIso.slice(0, 10)} ${afterVersionIso.slice(11, 19)}.999166+00`;
|
||||
await engine.executeRaw(`UPDATE pages SET updated_at = '${usUpdatedAt}'`);
|
||||
expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(2);
|
||||
|
||||
await runExtract(engine, ['--stale']);
|
||||
@@ -315,4 +318,48 @@ describe('gbrain extract --stale', () => {
|
||||
expect(exited).toBe(true);
|
||||
expect(msg).toContain('DB-source only');
|
||||
});
|
||||
|
||||
// ─── #2576 bug 1: --stale must run the same resolver passes as
|
||||
// `extract links --source db` ─────────────────────────────────────────────
|
||||
|
||||
test('#2576: bare wikilink resolves via global_basename on the --stale path', async () => {
|
||||
await engine.putPage('projects/struktura',
|
||||
{ type: 'project' as any, title: 'Struktura', compiled_truth: 'A project page.', timeline: '' });
|
||||
await engine.putPage('concepts/knowledge-graph',
|
||||
{ type: 'concept' as any, title: 'Knowledge Graph',
|
||||
compiled_truth: 'This concept relates to [[struktura]].', timeline: '' });
|
||||
await engine.setConfig('link_resolution.global_basename', 'true');
|
||||
try {
|
||||
await runExtract(engine, ['--stale']);
|
||||
} finally {
|
||||
await engine.setConfig('link_resolution.global_basename', 'false');
|
||||
}
|
||||
|
||||
// Pre-fix: the nullResolver (no resolveBasenameMatches) made
|
||||
// extractPageLinks skip the basename pass, so the sweep stamped the page
|
||||
// with the wikilink silently dropped — 0 links, watermark green.
|
||||
const links = await engine.getLinks('concepts/knowledge-graph');
|
||||
const strk = links.find(l => l.to_slug === 'projects/struktura');
|
||||
expect(strk).toBeDefined();
|
||||
expect(strk!.link_type).toBe('wikilink_basename');
|
||||
// Still stamped like every processed page.
|
||||
expect(await stampOf('concepts/knowledge-graph')).not.toBeNull();
|
||||
});
|
||||
|
||||
test('#2576: bare wikilink still drops on --stale when global_basename is OFF (back-compat)', async () => {
|
||||
await engine.putPage('projects/struktura',
|
||||
{ type: 'project' as any, title: 'Struktura', compiled_truth: 'A project page.', timeline: '' });
|
||||
await engine.putPage('concepts/knowledge-graph',
|
||||
{ type: 'concept' as any, title: 'Knowledge Graph',
|
||||
compiled_truth: 'This concept relates to [[struktura]].', timeline: '' });
|
||||
await engine.setConfig('link_resolution.global_basename', 'false');
|
||||
|
||||
await runExtract(engine, ['--stale']);
|
||||
|
||||
expect((await engine.getLinks('concepts/knowledge-graph'))).toHaveLength(0);
|
||||
// The gate lives in extractPageLinks opts now, not in a resolver swap —
|
||||
// the page is still stamped either way.
|
||||
expect(await stampOf('concepts/knowledge-graph')).not.toBeNull();
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user