From 5e8816e7d641840d1a99cc57cf2ec9659808d824 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 16:45:27 -0700 Subject: [PATCH] fix(links): resolve [[wikilink]] + slug-path frontmatter values; frontmatter-fresh incremental extract (#3087) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(links): resolve [[wikilink]] + slug-path frontmatter values; keep frontmatter links fresh on the incremental cycle Takeover/rebase of two community PRs: PR #1983 — frontmatter link fields never resolved Obsidian-style values: - makeResolver step 1's strict slug regex rejected digit-leading folders (90-people/nicolai) and nested paths (a/b/c); broadened to any slug-shaped value with an EXACT getPage match only (no fuzzy, no false positives). - extractFrontmatterLinks resolved "[[dir/slug]]" verbatim; new anchored unwrapWikilink() strips wholly-wrapped [[...]] (and |alias/#heading/^block) before resolution. Bare values pass through unchanged. - Same broadened slug-shape applied to the fs-path synthetic resolver in extractLinksFromFile (exact Set membership guards it), so the fs frontmatter path resolves PARA-numbered slugs too. PR #2434 — the cycle's incremental extract (extractForSlugs) extracted body links only, so externally-edited YAML (sources:/related:) edges drifted stale. Adds an includeFrontmatter opt (threaded as a param after sourceId, which master added in #1747/#1503 after the PR was cut), gated by the new config key autopilot.incremental_extract_include_frontmatter (default off, preserves body-only behavior). Tests: unwrapWikilink unit coverage, broadened-resolver + end-to-end frontmatter cases in test/link-extraction.test.ts; fs-resolver digit-leading case in test/extract.test.ts; incremental gate off/on cases in test/extract-incremental.test.ts. Co-authored-by: spiky02plateau Co-Authored-By: Claude Fable 5 * fix(cycle): honor DB-plane config for incremental_extract_include_frontmatter The gate read loadConfig() (file/env plane) only, but the documented enable command — gbrain config set autopilot.incremental_extract_include_frontmatter true — writes the DB plane via engine.setConfig, so the feature could never be turned on the documented way (silent no-op, #2120 class). Now the file plane wins when the key is present there; otherwise the DB plane is consulted, matching the autopilot.auto_drain.* read pattern. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Sinabina Co-authored-by: spiky02plateau Co-authored-by: Claude Fable 5 Co-authored-by: Garry Tan --- src/commands/extract.ts | 25 ++++- src/core/config.ts | 12 +++ src/core/cycle.ts | 16 +++ src/core/link-extraction.ts | 39 ++++++- test/extract-incremental.test.ts | 34 ++++++ test/extract.test.ts | 12 +++ test/link-extraction.test.ts | 173 +++++++++++++++++++++++++++++++ 7 files changed, 305 insertions(+), 6 deletions(-) diff --git a/src/commands/extract.ts b/src/commands/extract.ts index b2aabc5fd..75645ca58 100644 --- a/src/commands/extract.ts +++ b/src/commands/extract.ts @@ -433,7 +433,10 @@ export async function extractLinksFromFile( async resolve(name: string, dirHint?: string | string[]): Promise { if (!name) return null; const trimmed = name.trim(); - if (/^[a-z][a-z0-9-]*\/[a-z0-9][a-z0-9-]*$/.test(trimmed) && allSlugs.has(trimmed)) { + // Same broadened slug-shape as makeResolver step 1: accepts + // digit-leading folders (`90-people/nicolai`) and nested paths. + // Exact Set membership guards it — no false positives. + if (/\//.test(trimmed) && /^[a-z0-9][a-z0-9/_-]*$/.test(trimmed) && allSlugs.has(trimmed)) { return trimmed; } const hints = Array.isArray(dirHint) ? dirHint : (dirHint ? [dirHint] : []); @@ -582,6 +585,17 @@ export interface ExtractOpts { * before (single-'default'-source brains unaffected). */ sourceId?: string; + /** + * v0.42 — also extract frontmatter links on the incremental (slugs) path. + * `extractForSlugs` extracts BODY links only by default; set this true to also + * parse each changed page's frontmatter so `sources:`/`related:` edges stay fresh + * when YAML is edited externally and synced in. Applied PER changed page, so the + * incremental walk stays bounded (no switch to a full DB scan). Only honored on + * the incremental path (`slugs` defined); the full-walk path already covers + * frontmatter via its own dispatch. Gated upstream by the config key + * `autopilot.incremental_extract_include_frontmatter` (default off). + */ + includeFrontmatter?: boolean; } /** @@ -620,7 +634,7 @@ export async function runExtractCore(engine: BrainEngine, opts: ExtractOpts): Pr // Nothing changed — skip entirely. return result; } - const r = await extractForSlugs(engine, opts.dir, opts.slugs, opts.mode, dryRun, jsonMode, workers, opts.signal, opts.sourceId); + const r = await extractForSlugs(engine, opts.dir, opts.slugs, opts.mode, dryRun, jsonMode, workers, opts.signal, opts.sourceId, opts.includeFrontmatter); result.links_created = r.links_created; result.timeline_entries_created = r.timeline_created; result.pages_processed = r.pages; @@ -1011,6 +1025,11 @@ async function extractForSlugs( signal?: AbortSignal, // #1747/#1503: stamp resolved brain source id on batch rows (see ExtractOpts.sourceId). sourceId?: string, + // v0.42: when true, also extract frontmatter links per changed page so + // externally-edited YAML (`sources:`/`related:`) stays fresh on the cycle. + // Default false preserves the body-only incremental behavior. Gated upstream + // by `autopilot.incremental_extract_include_frontmatter`. + includeFrontmatter: boolean = false, ): Promise<{ links_created: number; timeline_created: number; pages: number }> { // Build the full slug set for link resolution (fast: just readdir, no file reads) const allFiles = walkMarkdownFiles(brainDir); @@ -1089,7 +1108,7 @@ async function extractForSlugs( const content = readFileSync(fullPath, 'utf-8'); if (doLinks) { - const links = await extractLinksFromFile(content, relPath, allSlugs, { globalBasename }); + const links = await extractLinksFromFile(content, relPath, allSlugs, { globalBasename, includeFrontmatter }); for (const link of links) { if (dryRun) { if (!jsonMode) console.log(` ${link.from_slug} → ${link.to_slug} (${link.link_type})`); diff --git a/src/core/config.ts b/src/core/config.ts index e2fdc971b..50fa2f723 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -146,6 +146,18 @@ export interface GBrainConfig { /** Daily spend cap (USD); bounds drains/day = floor(cap / ~$0.30). Default 2.0. */ max_usd_per_day?: number; }; + /** + * v0.42 — keep frontmatter links fresh on the incremental cycle. The cycle's + * extract phase re-extracts only the slugs a sync changed, but `extractForSlugs` + * extracts BODY links only — frontmatter (`sources:`/`related:` etc.) link edges + * silently drift stale when a page's YAML is edited externally and synced in. + * Set true to also extract frontmatter links per changed page each cycle, keeping + * externally-edited YAML edges fresh without a full rescan. Default false + * (preserves current behavior). Read via the file/env/DB plane in the cycle's + * extract dispatch. Disable/enable with + * `gbrain config set autopilot.incremental_extract_include_frontmatter `. + */ + incremental_extract_include_frontmatter?: boolean; }; eval?: { /** false disables capture entirely. Defaults to true. */ diff --git a/src/core/cycle.ts b/src/core/cycle.ts index a6a15fbc1..f98e370e8 100644 --- a/src/core/cycle.ts +++ b/src/core/cycle.ts @@ -1013,6 +1013,21 @@ async function runPhaseExtract( ): Promise { try { const { runExtractCore } = await import('../commands/extract.ts'); + const { loadConfig } = await import('./config.ts'); + // Default off: the incremental cycle extracts body links only unless the + // operator opts in to keeping externally-edited frontmatter links fresh too. + // Both planes, file wins (env > file > DB precedence, per loadConfigWithEngine): + // `gbrain config set autopilot.incremental_extract_include_frontmatter true` + // writes the DB plane (engine.setConfig), so a file-plane-only read here + // would make the documented enable command a silent no-op (#2120 class). + const fileVal = loadConfig()?.autopilot?.incremental_extract_include_frontmatter; + let includeFrontmatter = fileVal === true; + if (fileVal === undefined) { + try { + includeFrontmatter = + (await engine.getConfig('autopilot.incremental_extract_include_frontmatter')) === 'true'; + } catch { /* config table unreadable → default off */ } + } // Extract is read-mostly against the filesystem + write to links table. // Honor dryRun by skipping with a 'skipped' entry: extract doesn't have // a clean dry-run mode today and runCycle should be honest about it. @@ -1033,6 +1048,7 @@ async function runPhaseExtract( slugs: changedSlugs, // undefined = full walk (first run / manual) signal, sourceId, + includeFrontmatter, // honored on the incremental (slugs) path only }); const linksCreated = result?.links_created ?? 0; const timelineCreated = result?.timeline_entries_created ?? 0; diff --git a/src/core/link-extraction.ts b/src/core/link-extraction.ts index 4b8300d3c..41451abc3 100644 --- a/src/core/link-extraction.ts +++ b/src/core/link-extraction.ts @@ -960,8 +960,17 @@ export function makeResolver( const hints = Array.isArray(dirHint) ? dirHint : (dirHint ? [dirHint] : []); - // Step 1: already a slug? (dir/name shape, lowercase, hyphenated) - if (/^[a-z][a-z0-9-]*\/[a-z0-9][a-z0-9-]*$/.test(trimmed)) { + // Step 1: already a slug? Try an exact page lookup for any slug-shaped + // value (contains '/', slug charset). Broadened beyond the original + // single-segment lowercase-leading form (`^[a-z][a-z0-9-]*\/[a-z0-9]...`) + // to also accept digit-leading folders (`90-people/nicolai`, + // `01-trading/...`) and nested paths (`a/b/c`) — common in PARA-numbered + // vaults. This is an EXACT getPage match only — no fuzzy — so it never + // produces a false positive; a non-existent slug just falls through to + // the steps below. Fixes frontmatter `related: [[dir/slug]]` values + // (unwrapped by unwrapWikilink) that name a real page the strict regex + // could not reach and whose full-path fuzzy score is below threshold. + if (/\//.test(trimmed) && /^[a-z0-9][a-z0-9/_-]*$/.test(trimmed)) { const page = await engine.getPage(trimmed); if (page) { cache.set(cacheKey, trimmed); @@ -1025,6 +1034,25 @@ export function makeResolver( // ─── Frontmatter extractor ────────────────────────────────────── +/** + * Unwrap an Obsidian `[[wikilink]]` frontmatter value to its bare link + * target so the resolver (which expects bare titles / dir slugs) can match + * it. Mainstream Obsidian authors frontmatter links as `related: ["[[Page]]"]`; + * without this, the resolver treats the brackets as part of the value and a + * `[[90-people/nicolai]]` is normalized into `90peoplenicolai`, so it never + * resolves. Strips a trailing `|alias`, `#heading`, or `^block` suffix — the + * link target only. The regex is anchored to a wholly-wrapped value + * (`^\s*\[\[…\]\]\s*$`), so bare titles and any value not fully wrapped pass + * through unchanged and existing behavior is preserved exactly. + */ +export function unwrapWikilink(value: string): string { + const match = /^\s*\[\[(.+?)\]\]\s*$/.exec(value); + if (!match) return value; + // Take the link target: drop |alias, then #heading / ^block suffixes. + const target = match[1].split('|')[0].split('#')[0].split('^')[0]; + return target.trim(); +} + export interface UnresolvedFrontmatterRef { /** The frontmatter field name. */ field: string; @@ -1082,7 +1110,12 @@ export async function extractFrontmatterLinks( } if (!name) continue; // skip numbers, nulls, malformed objects - const resolved = await resolver.resolve(name, mapping.dirHint); + // Accept Obsidian `[[wikilink]]` values in frontmatter link fields by + // unwrapping to the bare target before resolution. Bare titles pass + // through unchanged; the original `name` is preserved for the + // unresolved report and edge context. + const linkTarget = unwrapWikilink(name); + const resolved = await resolver.resolve(linkTarget, mapping.dirHint); if (!resolved) { unresolved.push({ field, name }); continue; diff --git a/test/extract-incremental.test.ts b/test/extract-incremental.test.ts index dc47069f6..21307bb8a 100644 --- a/test/extract-incremental.test.ts +++ b/test/extract-incremental.test.ts @@ -236,3 +236,37 @@ describe('runExtractCore — incremental cycle path (#417)', () => { expect(result.links_created).toBeGreaterThan(0); }); }); +describe('runExtractCore — incremental frontmatter gate (includeFrontmatter)', () => { + // alice has a `source:` frontmatter edge but NO body links. The incremental + // path extracts body links only by default, so the frontmatter edge is the + // sole signal that distinguishes the gate off vs on. + const aliceFm = '---\nsource: companies/acme-example\n---\n# alice'; + + test('9. default (flag omitted) does NOT extract frontmatter links on the incremental path', async () => { + await seedPage('companies/acme-example', '# acme'); + await seedPage('people/alice-example', aliceFm); + const result = await runExtractCore(engine as unknown as BrainEngine, { + mode: 'all', + dir: tempDir, + slugs: ['people/alice-example'], + }); + // alice's only potential edge is her frontmatter `source:`; with the gate off + // it must not be extracted (preserves the body-only incremental behavior). + expect(result.pages_processed).toBe(1); + expect(result.links_created).toBe(0); + }); + + test('10. includeFrontmatter: true extracts the frontmatter link on the incremental path', async () => { + await seedPage('companies/acme-example', '# acme'); + await seedPage('people/alice-example', aliceFm); + const result = await runExtractCore(engine as unknown as BrainEngine, { + mode: 'all', + dir: tempDir, + slugs: ['people/alice-example'], + includeFrontmatter: true, + }); + // Same page, gate on → the `source:` frontmatter edge is now extracted. + expect(result.pages_processed).toBe(1); + expect(result.links_created).toBeGreaterThan(0); + }); +}); diff --git a/test/extract.test.ts b/test/extract.test.ts index 5764de52b..90cf54d1f 100644 --- a/test/extract.test.ts +++ b/test/extract.test.ts @@ -76,6 +76,18 @@ describe('extractLinksFromFile', () => { } }); + it('resolves wrapped [[wikilink]] digit-leading slug-path in frontmatter (fs resolver, broadened step 1)', async () => { + // Same bug class as makeResolver step 1 (#1983): the fs resolver's strict + // `^[a-z]…` slug regex rejected digit-leading / nested paths, so a PARA-vault + // `related: "[[90-people/nicolai]]"` never resolved even though the page exists. + const content = '---\nrelated: "[[90-people/nicolai]]"\ntype: concept\n---\nContent.'; + const allSlugs = new Set(['wiki/note', '90-people/nicolai']); + const links = await extractLinksFromFile(content, 'wiki/note.md', allSlugs, { includeFrontmatter: true }); + const related = links.filter(l => l.link_type === 'related_to'); + expect(related).toHaveLength(1); + expect(related[0].to_slug).toBe('90-people/nicolai'); + }); + it('frontmatter extraction is default OFF (back-compat)', async () => { // Without includeFrontmatter, fs-source no longer auto-extracts frontmatter. // Matches db-source behavior. User opts in with --include-frontmatter flag. diff --git a/test/link-extraction.test.ts b/test/link-extraction.test.ts index 9a2bc4f7d..bab980431 100644 --- a/test/link-extraction.test.ts +++ b/test/link-extraction.test.ts @@ -9,6 +9,7 @@ import { parseTimelineEntries, isAutoLinkEnabled, FRONTMATTER_LINK_MAP, + unwrapWikilink, type SlugResolver, } from '../src/core/link-extraction.ts'; import type { BrainEngine } from '../src/core/engine.ts'; @@ -1399,3 +1400,175 @@ describe('parseTimelineEntries — Format 3: inline [Source: ..., YYYY-MM-DD] ci expect(parseTimelineEntries('[Source: import batch, 2025-07-01]')).toHaveLength(0); }); }); +// ─── Frontmatter [[wikilink]] + slug-path resolution ────────────────────── +// Mainstream Obsidian authors frontmatter links as `related: ["[[Page]]"]`, +// and PARA-numbered vaults use digit-leading / nested slug paths like +// `[[90-people/nicolai]]`. Both were silently dropped: brackets were treated +// as part of the value and the step-1 slug regex (`^[a-z]…`) rejected +// digit-leading / nested paths, while full-path fuzzy scored below threshold. +// Fix: unwrapWikilink() before resolution + an exact getPage() for any +// slug-shaped value (exact-match only → no false positives). + +describe('unwrapWikilink', () => { + test('wrapped title → bare title', () => { + expect(unwrapWikilink('[[Monday Range]]')).toBe('Monday Range'); + }); + test('wrapped slug-path (digit-leading folder) → bare slug', () => { + expect(unwrapWikilink('[[90-people/nicolai]]')).toBe('90-people/nicolai'); + }); + test('wrapped nested slug-path → bare slug', () => { + expect(unwrapWikilink('[[01-trading/wiki/strategies/opening-range-breakout]]')) + .toBe('01-trading/wiki/strategies/opening-range-breakout'); + }); + test('strips |alias', () => { + expect(unwrapWikilink('[[90-people/nicolai|Nicolai]]')).toBe('90-people/nicolai'); + }); + test('strips #heading', () => { + expect(unwrapWikilink('[[Page#Section]]')).toBe('Page'); + }); + test('strips ^block', () => { + expect(unwrapWikilink('[[Page^abc123]]')).toBe('Page'); + }); + test('surrounding whitespace tolerated', () => { + expect(unwrapWikilink(' [[Page]] ')).toBe('Page'); + }); + test('bare title passes through unchanged', () => { + expect(unwrapWikilink('Monday Range')).toBe('Monday Range'); + }); + test('bare slug passes through unchanged', () => { + expect(unwrapWikilink('90-people/nicolai')).toBe('90-people/nicolai'); + }); + test('partially-wrapped value is NOT unwrapped (anchored)', () => { + // Not a wholly-wrapped value → left intact so existing behavior is exact. + expect(unwrapWikilink('see [[Page]] for detail')).toBe('see [[Page]] for detail'); + }); +}); + +describe('makeResolver — slug-path exact getPage (step 1 broadened)', () => { + function fakeEngine( + slugs: string[], + fuzzyMap: Map = new Map(), + ): BrainEngine { + const lookup = new Set(slugs); + return { + async getPage(slug: string) { return lookup.has(slug) ? { slug } as any : null; }, + async findByTitleFuzzy(name: string) { return fuzzyMap.get(name) ?? null; }, + async searchKeyword() { return []; }, + } as unknown as BrainEngine; + } + + test('digit-leading folder slug resolves via exact getPage', async () => { + const r = makeResolver(fakeEngine(['90-people/nicolai'])); + expect(await r.resolve('90-people/nicolai')).toBe('90-people/nicolai'); + }); + + test('nested (>2 segment) slug resolves via exact getPage', async () => { + const r = makeResolver(fakeEngine(['01-trading/wiki/strategies/opening-range-breakout'])); + expect(await r.resolve('01-trading/wiki/strategies/opening-range-breakout')) + .toBe('01-trading/wiki/strategies/opening-range-breakout'); + }); + + test('regression: single-segment lowercase slug still resolves', async () => { + const r = makeResolver(fakeEngine(['people/pedro'])); + expect(await r.resolve('people/pedro')).toBe('people/pedro'); + }); + + test('exact-only: slug-shaped value with no matching page falls through (no false positive)', async () => { + // `90-people/ghost` is slug-shaped but absent → step-1 getPage misses, + // no fuzzy hit → null. Never invents an edge. + const r = makeResolver(fakeEngine(['90-people/nicolai'])); + expect(await r.resolve('90-people/ghost')).toBeNull(); + }); + + test('non-slug value still routes to fuzzy', async () => { + const r = makeResolver(fakeEngine( + ['01-trading/monday-range'], + new Map([['Monday Range', { slug: '01-trading/monday-range', similarity: 1 }]]), + )); + expect(await r.resolve('Monday Range')).toBe('01-trading/monday-range'); + }); +}); + +describe('extractFrontmatterLinks — [[wikilink]] related: values (end-to-end)', () => { + function fakeEngine( + slugs: string[], + fuzzyMap: Map = new Map(), + ): BrainEngine { + const lookup = new Set(slugs); + return { + async getPage(slug: string) { return lookup.has(slug) ? { slug } as any : null; }, + async findByTitleFuzzy(name: string) { return fuzzyMap.get(name) ?? null; }, + async searchKeyword() { return []; }, + } as unknown as BrainEngine; + } + + test('wrapped slug-path related: resolves (the core win)', async () => { + const resolver = makeResolver(fakeEngine(['90-people/nicolai'])); + const { candidates, unresolved } = await extractFrontmatterLinks( + 'wiki/originals/ideas/note', 'note' as never, + { related: '[[90-people/nicolai]]' }, resolver, + ); + expect(unresolved).toHaveLength(0); + expect(candidates).toHaveLength(1); + expect(candidates[0]).toMatchObject({ + fromSlug: 'wiki/originals/ideas/note', + targetSlug: '90-people/nicolai', + linkType: 'related_to', + linkSource: 'frontmatter', + }); + }); + + test('wrapped nested slug-path related: resolves', async () => { + const resolver = makeResolver(fakeEngine(['01-trading/wiki/strategies/opening-range-breakout'])); + const { candidates } = await extractFrontmatterLinks( + 'wiki/note', 'note' as never, + { related: ['[[01-trading/wiki/strategies/opening-range-breakout]]'] }, resolver, + ); + expect(candidates).toHaveLength(1); + expect(candidates[0].targetSlug).toBe('01-trading/wiki/strategies/opening-range-breakout'); + }); + + test('wrapped value with |alias resolves to the target', async () => { + const resolver = makeResolver(fakeEngine(['90-people/nicolai'])); + const { candidates } = await extractFrontmatterLinks( + 'wiki/note', 'note' as never, + { related: '[[90-people/nicolai|Nicolai]]' }, resolver, + ); + expect(candidates).toHaveLength(1); + expect(candidates[0].targetSlug).toBe('90-people/nicolai'); + }); + + test('regression: bare slug related: still resolves', async () => { + const resolver = makeResolver(fakeEngine(['90-people/nicolai'])); + const { candidates } = await extractFrontmatterLinks( + 'wiki/note', 'note' as never, + { related: '90-people/nicolai' }, resolver, + ); + expect(candidates).toHaveLength(1); + expect(candidates[0].targetSlug).toBe('90-people/nicolai'); + }); + + test('regression: wrapped title resolves via fuzzy (brackets harmless)', async () => { + const resolver = makeResolver(fakeEngine( + ['01-trading/monday-range'], + new Map([['Monday Range', { slug: '01-trading/monday-range', similarity: 1 }]]), + )); + const { candidates } = await extractFrontmatterLinks( + 'wiki/note', 'note' as never, + { related: '[[Monday Range]]' }, resolver, + ); + expect(candidates).toHaveLength(1); + expect(candidates[0].targetSlug).toBe('01-trading/monday-range'); + }); + + test('unknown wrapped slug → unresolved (no crash), original value preserved', async () => { + const resolver = makeResolver(fakeEngine(['90-people/nicolai'])); + const { candidates, unresolved } = await extractFrontmatterLinks( + 'wiki/note', 'note' as never, + { related: '[[99-archive/does-not-exist]]' }, resolver, + ); + expect(candidates).toHaveLength(0); + expect(unresolved).toHaveLength(1); + expect(unresolved[0]).toEqual({ field: 'related', name: '[[99-archive/does-not-exist]]' }); + }); +});