mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
fix(extract): let custom schema packs drive plain link extraction (#3190)
Three gates made custom packs decorative for gbrain extract links / put_page auto-link: 1. DIR_PATTERN whitelist: entity regexes now union in the active pack's page_types[].path_prefixes (slug-safe, longest-first). 2. Verb inference: extractPageLinks consults pack link_types via inferLinkTypeFromPack (under a per-page ReDoS budget) before the legacy matchers, exactly the wrap link-inference.ts documents. 3. frontmatter_links: pack mappings extend FRONTMATTER_LINK_MAP at extraction time (outgoing, pack entity dirs as resolution hints); the hardcoded map wins on any (pageType, field) it already covers so a pack restating built-ins cannot emit reversed/duplicate edges. Codegen'd base packs (gbrain-base, gbrain-base-v2) are excluded — their semantics ARE the in-code tables; consulting their sketch regexes would double-apply worse copies. Threaded through extract links --source db, extract --stale, and put_page runAutoLink via loadActivePackBestEffort. LINK_EXTRACTOR_VERSION_TS bumped so stale sweeps re-extract. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
c428500a4c
commit
91674d59c2
+10
-1
@@ -43,6 +43,8 @@ import {
|
||||
} from '../core/link-extraction.ts';
|
||||
import { createProgress } from '../core/progress.ts';
|
||||
import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
|
||||
import { loadActivePackBestEffort } from '../core/schema-pack/best-effort.ts';
|
||||
import { packLinkExtractionView } from '../core/schema-pack/link-inference.ts';
|
||||
import { pathToSlug, pruneDir, isSyncable } from '../core/sync.ts';
|
||||
// v0.41.18.0: withRetry + isRetryableConnError + WithRetryOpts moved to
|
||||
// src/core/retry.ts as the canonical primitive. Engine methods
|
||||
@@ -1381,6 +1383,9 @@ async function extractLinksFromDB(
|
||||
// Issue #972: opt-in global-basename wikilink resolution. Read once
|
||||
// per extract run; threaded into each extractPageLinks call.
|
||||
const globalBasename = await isGlobalBasenameEnabled(engine);
|
||||
// #3190: active custom pack (null for the codegen'd base packs) so pack
|
||||
// path_prefixes / link verbs / frontmatter_links drive extraction.
|
||||
const pack = packLinkExtractionView(await loadActivePackBestEffort({ engine } as never));
|
||||
// v0.32.8: listAllPageRefs enumerates (slug, source_id) so we can thread
|
||||
// sourceId to getPage AND build a cross-source resolution map for link
|
||||
// disambiguation. Pre-fix used getAllSlugs() which collapsed
|
||||
@@ -1461,7 +1466,7 @@ async function extractLinksFromDB(
|
||||
// basename lookup; off by default for back-compat.
|
||||
const extracted = await extractPageLinks(
|
||||
slug, fullContent, page.frontmatter, page.type, resolver,
|
||||
{ skipFrontmatter: !includeFrontmatter, globalBasename },
|
||||
{ skipFrontmatter: !includeFrontmatter, globalBasename, pack },
|
||||
);
|
||||
unresolved.push(...extracted.unresolved);
|
||||
|
||||
@@ -1687,6 +1692,9 @@ export async function extractStaleFromDB(
|
||||
const resolver = makeResolver(engine, { mode: 'batch' });
|
||||
const nullResolver = { resolve: async () => null as string | null };
|
||||
const activeResolver = includeFrontmatter ? resolver : nullResolver;
|
||||
// #3190: same pack threading as extractLinksFromDB so `extract --stale`
|
||||
// produces the same edges as a manual `extract links --source db`.
|
||||
const pack = packLinkExtractionView(await loadActivePackBestEffort({ engine } as never));
|
||||
const allRefs = await engine.listAllPageRefs();
|
||||
const allSlugs = new Set<string>();
|
||||
const slugToSources = new Map<string, string[]>();
|
||||
@@ -1719,6 +1727,7 @@ export async function extractStaleFromDB(
|
||||
const fullContent = page.compiled_truth + '\n' + page.timeline;
|
||||
const extracted = await extractPageLinks(
|
||||
page.slug, fullContent, page.frontmatter, page.type, activeResolver,
|
||||
{ pack },
|
||||
);
|
||||
for (const c of extracted.candidates) {
|
||||
const r = resolveCandidateSources(c, page.slug, page.source_id, allSlugs, slugToSources);
|
||||
|
||||
+88
-23
@@ -14,6 +14,8 @@
|
||||
import type { BrainEngine } from './engine.ts';
|
||||
import type { PageType } from './types.ts';
|
||||
import { ensureWellFormed } from './text-safe.ts';
|
||||
import { inferLinkTypeFromPack, type PackLinkExtraction } from './schema-pack/link-inference.ts';
|
||||
import { PageRegexBudget } from './schema-pack/redos-guard.ts';
|
||||
|
||||
/**
|
||||
* v0.42.7 — link-extraction version stamp. Bump this ISO timestamp whenever the
|
||||
@@ -28,7 +30,7 @@ import { ensureWellFormed } from './text-safe.ts';
|
||||
* OR updated_at > links_extracted_at`. It is an ISO-8601 string (NOT a number) —
|
||||
* the column is TIMESTAMPTZ and the predicate binds it as `::timestamptz`.
|
||||
*/
|
||||
export const LINK_EXTRACTOR_VERSION_TS = '2026-05-31T00:00:00Z';
|
||||
export const LINK_EXTRACTOR_VERSION_TS = '2026-07-23T00:00:00Z';
|
||||
|
||||
// ─── Entity references ──────────────────────────────────────────
|
||||
|
||||
@@ -83,7 +85,22 @@ export type LinkResolutionType = 'qualified' | 'unqualified';
|
||||
* - Our domain extensions: tech, finance, personal, openclaw (domain-organized wikis)
|
||||
* - Our entity prefix: entities (we kept some legacy entities/projects/ pages)
|
||||
*/
|
||||
const DIR_PATTERN = '(?:people|companies|meetings|concepts|deal|civic|project|projects|source|media|yc|tech|finance|personal|openclaw|entities)';
|
||||
const DIR_ALTERNATIVES = 'people|companies|meetings|concepts|deal|civic|project|projects|source|media|yc|tech|finance|personal|openclaw|entities';
|
||||
const DIR_PATTERN = `(?:${DIR_ALTERNATIVES})`;
|
||||
|
||||
/**
|
||||
* #3190 — union the built-in dir whitelist with pack-declared entity dirs
|
||||
* (from `page_types[].path_prefixes`, pre-validated to a slug-safe charset
|
||||
* by `packLinkExtractionView`). Longer alternatives sort first so a nested
|
||||
* prefix like `wiki/people` wins over a hypothetical `wiki`.
|
||||
*/
|
||||
function dirPatternWith(extraDirs?: string[]): string {
|
||||
if (!extraDirs || extraDirs.length === 0) return DIR_PATTERN;
|
||||
const extras = [...new Set(extraDirs)]
|
||||
.sort((a, b) => (b.length - a.length) || a.localeCompare(b))
|
||||
.map(d => d.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'));
|
||||
return `(?:${extras.join('|')}|${DIR_ALTERNATIVES})`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Match `[Name](path)` markdown links pointing to entity directories.
|
||||
@@ -95,8 +112,8 @@ const DIR_PATTERN = '(?:people|companies|meetings|concepts|deal|civic|project|pr
|
||||
* The regex permits an optional `../` prefix (any number) and an optional
|
||||
* `.md` suffix so the same function works for both filesystem and DB content.
|
||||
*/
|
||||
const ENTITY_REF_RE = new RegExp(
|
||||
`\\[([^\\]]+)\\]\\((?:\\.\\.\\/)*(${DIR_PATTERN}\\/[^)\\s]+?)(?:\\.md)?\\)`,
|
||||
const entityRefRe = (dirPattern: string) => new RegExp(
|
||||
`\\[([^\\]]+)\\]\\((?:\\.\\.\\/)*(${dirPattern}\\/[^)\\s]+?)(?:\\.md)?\\)`,
|
||||
'g',
|
||||
);
|
||||
|
||||
@@ -104,12 +121,12 @@ const ENTITY_REF_RE = new RegExp(
|
||||
* Match Obsidian-style `[[path]]` or `[[path|Display Text]]` wikilinks.
|
||||
* Captures: slug (dir/...), displayName (optional).
|
||||
*
|
||||
* Same dir whitelist as ENTITY_REF_RE. Strips trailing `.md`, strips section
|
||||
* Same dir whitelist as entityRefRe. Strips trailing `.md`, strips section
|
||||
* anchors (`#heading`), skips external URLs. Wiki KBs use this format almost
|
||||
* exclusively so missing it leaves the graph empty.
|
||||
*/
|
||||
const WIKILINK_RE = new RegExp(
|
||||
`\\[\\[(${DIR_PATTERN}\\/[^|\\]#]+?)(?:#[^|\\]]*?)?(?:\\|([^\\]]+?))?\\]\\]`,
|
||||
const wikilinkRe = (dirPattern: string) => new RegExp(
|
||||
`\\[\\[(${dirPattern}\\/[^|\\]#]+?)(?:#[^|\\]]*?)?(?:\\|([^\\]]+?))?\\]\\]`,
|
||||
'g',
|
||||
);
|
||||
|
||||
@@ -121,12 +138,12 @@ const WIKILINK_RE = new RegExp(
|
||||
*
|
||||
* Captures: sourceId, slug (dir/...), displayName (optional).
|
||||
*
|
||||
* Matched BEFORE WIKILINK_RE so `[[wiki:topics/ai]]` isn't mis-parsed by
|
||||
* the unqualified regex (the source prefix would not satisfy DIR_PATTERN
|
||||
* anyway, but the two-pass approach keeps intent crystal-clear).
|
||||
* Matched BEFORE the unqualified wikilink pass so `[[wiki:topics/ai]]` isn't
|
||||
* mis-parsed by the unqualified regex (the source prefix would not satisfy
|
||||
* DIR_PATTERN anyway, but the two-pass approach keeps intent crystal-clear).
|
||||
*/
|
||||
const QUALIFIED_WIKILINK_RE = new RegExp(
|
||||
`\\[\\[([a-z0-9](?:[a-z0-9-]{0,30}[a-z0-9])?):(${DIR_PATTERN}\\/[^|\\]#]+?)(?:#[^|\\]]*?)?(?:\\|([^\\]]+?))?\\]\\]`,
|
||||
const qualifiedWikilinkRe = (dirPattern: string) => new RegExp(
|
||||
`\\[\\[([a-z0-9](?:[a-z0-9-]{0,30}[a-z0-9])?):(${dirPattern}\\/[^|\\]#]+?)(?:#[^|\\]]*?)?(?:\\|([^\\]]+?))?\\]\\]`,
|
||||
'g',
|
||||
);
|
||||
|
||||
@@ -298,17 +315,19 @@ export function extractCodeRefs(content: string): CodeRef[] {
|
||||
* here; caller dedups). Slugs appearing inside fenced or inline code blocks
|
||||
* are excluded — those are typically code samples, not real entity references.
|
||||
*/
|
||||
export function extractEntityRefs(content: string): EntityRef[] {
|
||||
export function extractEntityRefs(content: string, extraDirs?: string[]): EntityRef[] {
|
||||
const stripped = stripCodeBlocks(content);
|
||||
const refs: EntityRef[] = [];
|
||||
let match: RegExpExecArray | null;
|
||||
// #3190: pack-declared entity dirs widen the whitelist for this call.
|
||||
const dirPattern = dirPatternWith(extraDirs);
|
||||
|
||||
// 1. Markdown links: [Name](path)
|
||||
// Markdown links have no source-qualification syntax — they're
|
||||
// always unqualified. Omit sourceId so the shape stays compatible
|
||||
// with pre-v0.17 consumers doing strict equality.
|
||||
const markdownRanges: Array<[number, number]> = [];
|
||||
const mdPattern = new RegExp(ENTITY_REF_RE.source, ENTITY_REF_RE.flags);
|
||||
const mdPattern = entityRefRe(dirPattern);
|
||||
while ((match = mdPattern.exec(stripped)) !== null) {
|
||||
const name = match[1];
|
||||
const fullPath = match[2];
|
||||
@@ -322,7 +341,7 @@ export function extractEntityRefs(content: string): EntityRef[] {
|
||||
// Must run BEFORE the unqualified pass or we'd double-emit. We also
|
||||
// mask out the matched spans so pass 2b can't grab them.
|
||||
const qualifiedRanges: Array<[number, number]> = [];
|
||||
const qualPattern = new RegExp(QUALIFIED_WIKILINK_RE.source, QUALIFIED_WIKILINK_RE.flags);
|
||||
const qualPattern = qualifiedWikilinkRe(dirPattern);
|
||||
while ((match = qualPattern.exec(stripped)) !== null) {
|
||||
const sourceId = match[1];
|
||||
let slug = match[2].trim();
|
||||
@@ -339,7 +358,7 @@ export function extractEntityRefs(content: string): EntityRef[] {
|
||||
// Same shape rule: omit sourceId when unqualified.
|
||||
const unqualifiedRanges: Array<[number, number]> = [];
|
||||
const unmasked = maskRanges(stripped, qualifiedRanges);
|
||||
const wikiPattern = new RegExp(WIKILINK_RE.source, WIKILINK_RE.flags);
|
||||
const wikiPattern = wikilinkRe(dirPattern);
|
||||
while ((match = wikiPattern.exec(unmasked)) !== null) {
|
||||
let slug = match[1].trim();
|
||||
if (!slug) continue;
|
||||
@@ -468,12 +487,35 @@ export async function extractPageLinks(
|
||||
frontmatter: Record<string, unknown>,
|
||||
pageType: PageType,
|
||||
resolver: SlugResolver,
|
||||
opts: { globalBasename?: boolean; skipFrontmatter?: boolean } = {},
|
||||
opts: { globalBasename?: boolean; skipFrontmatter?: boolean; pack?: PackLinkExtraction | null } = {},
|
||||
): Promise<PageLinksResult> {
|
||||
const candidates: LinkCandidate[] = [];
|
||||
|
||||
// #3190: pack-aware extraction. When the caller threads the active
|
||||
// (non-base) pack, three things widen:
|
||||
// - entity-dir whitelist unions in pack path_prefixes,
|
||||
// - link-verb inference consults pack link_types BEFORE the legacy
|
||||
// matchers (the documented wrap in schema-pack/link-inference.ts),
|
||||
// - frontmatter_links mappings extend FRONTMATTER_LINK_MAP.
|
||||
// No pack (or a codegen'd base pack) → behavior is byte-identical to
|
||||
// the legacy path.
|
||||
const pack = opts.pack ?? null;
|
||||
const extraDirs = pack?.entity_dirs;
|
||||
// One ReDoS budget per page, shared across every per-candidate inference
|
||||
// call (mirrors extract-ner's per-page budget).
|
||||
const regexBudget = pack ? new PageRegexBudget() : null;
|
||||
const inferType = (context: string, targetSlug: string): string => {
|
||||
if (pack) {
|
||||
try {
|
||||
const verb = inferLinkTypeFromPack(pack, pageType as string, context, regexBudget ?? undefined);
|
||||
if (verb) return verb;
|
||||
} catch { /* pack inference is best-effort; fall through to legacy */ }
|
||||
}
|
||||
return inferLinkType(pageType, context, content, targetSlug);
|
||||
};
|
||||
|
||||
// 1. Markdown entity refs.
|
||||
for (const ref of extractEntityRefs(content)) {
|
||||
for (const ref of extractEntityRefs(content, extraDirs)) {
|
||||
// Issue #972: refs from the generic `[[bare-name]]` pass carry the
|
||||
// literal wikilink text, not a real page slug. When global_basename
|
||||
// mode is on AND the resolver implements basename lookup, resolve
|
||||
@@ -529,7 +571,7 @@ export async function extractPageLinks(
|
||||
const context = idx >= 0 ? excerpt(content, idx, 240) : ref.name;
|
||||
candidates.push({
|
||||
targetSlug: ref.slug,
|
||||
linkType: inferLinkType(pageType, context, content, ref.slug),
|
||||
linkType: inferType(context, ref.slug),
|
||||
context,
|
||||
linkSource: 'markdown',
|
||||
});
|
||||
@@ -540,7 +582,7 @@ export async function extractPageLinks(
|
||||
// Code blocks are stripped first — slugs in code samples are not real refs.
|
||||
const strippedContent = stripCodeBlocks(content);
|
||||
const bareRe = new RegExp(
|
||||
`\\b(${DIR_PATTERN}\\/[a-z0-9][a-z0-9/-]*[a-z0-9])\\b`,
|
||||
`\\b(${dirPatternWith(extraDirs)}\\/[a-z0-9][a-z0-9/-]*[a-z0-9])\\b`,
|
||||
'g',
|
||||
);
|
||||
let m: RegExpExecArray | null;
|
||||
@@ -551,7 +593,7 @@ export async function extractPageLinks(
|
||||
const context = excerpt(strippedContent, m.index, 240);
|
||||
candidates.push({
|
||||
targetSlug: m[1],
|
||||
linkType: inferLinkType(pageType, context, content, m[1]),
|
||||
linkType: inferType(context, m[1]),
|
||||
context,
|
||||
linkSource: 'markdown',
|
||||
});
|
||||
@@ -567,7 +609,7 @@ export async function extractPageLinks(
|
||||
// path needed `resolveBasenameMatches` on the real resolver.
|
||||
let fmUnresolved: UnresolvedFrontmatterRef[] = [];
|
||||
if (!opts.skipFrontmatter) {
|
||||
const fm = await extractFrontmatterLinks(slug, pageType, frontmatter, resolver);
|
||||
const fm = await extractFrontmatterLinks(slug, pageType, frontmatter, resolver, pack);
|
||||
candidates.push(...fm.candidates);
|
||||
fmUnresolved = fm.unresolved;
|
||||
}
|
||||
@@ -1047,11 +1089,34 @@ export async function extractFrontmatterLinks(
|
||||
pageType: PageType,
|
||||
frontmatter: Record<string, unknown>,
|
||||
resolver: SlugResolver,
|
||||
pack?: PackLinkExtraction | null,
|
||||
): Promise<FrontmatterExtractResult> {
|
||||
const candidates: LinkCandidate[] = [];
|
||||
const unresolved: UnresolvedFrontmatterRef[] = [];
|
||||
|
||||
for (const mapping of FRONTMATTER_LINK_MAP) {
|
||||
// #3190: pack-declared frontmatter_links extend the hardcoded map.
|
||||
// The hardcoded map WINS on any (pageType, field) it already covers —
|
||||
// it carries direction + dir hints the pack schema doesn't — so a pack
|
||||
// restating a built-in field can't emit reversed/duplicate edges.
|
||||
// Pack mappings are outgoing (page → resolved value); the pack's
|
||||
// entity dirs serve as resolution hints so bare-name values resolve
|
||||
// into pack-declared layouts (e.g. `wiki/people/<slug>`).
|
||||
const mappings: FrontmatterFieldMapping[] = [...FRONTMATTER_LINK_MAP];
|
||||
for (const fl of pack?.frontmatter_links ?? []) {
|
||||
const freeFields = fl.fields.filter(f =>
|
||||
!FRONTMATTER_LINK_MAP.some(m =>
|
||||
m.fields.includes(f) && (m.pageType === undefined || m.pageType === fl.page_type)));
|
||||
if (freeFields.length === 0) continue;
|
||||
mappings.push({
|
||||
fields: freeFields,
|
||||
pageType: fl.page_type,
|
||||
type: fl.link_type,
|
||||
direction: 'outgoing',
|
||||
dirHint: pack?.entity_dirs ?? '',
|
||||
});
|
||||
}
|
||||
|
||||
for (const mapping of mappings) {
|
||||
if (mapping.pageType && mapping.pageType !== pageType) continue;
|
||||
for (const field of mapping.fields) {
|
||||
const value = frontmatter[field];
|
||||
|
||||
+10
-1
@@ -1161,9 +1161,18 @@ async function runAutoLink(
|
||||
const resolver = makeResolver(engine, { mode: 'live', sourceId: opts?.sourceId });
|
||||
// Issue #972: opt-in bare-wikilink basename resolution. Off by default.
|
||||
const globalBasename = await isGlobalBasenameEnabled(engine);
|
||||
// #3190: consult the active custom pack (path_prefixes / link verbs /
|
||||
// frontmatter_links) so put_page auto-link matches what `gbrain extract
|
||||
// links --source db` produces. Best-effort: null for base packs or on
|
||||
// any load failure.
|
||||
const { loadActivePackBestEffort } = await import('./schema-pack/best-effort.ts');
|
||||
const { packLinkExtractionView } = await import('./schema-pack/link-inference.ts');
|
||||
const pack = packLinkExtractionView(
|
||||
await loadActivePackBestEffort({ engine, sourceId: opts?.sourceId } as never),
|
||||
);
|
||||
const { candidates, unresolved } = await extractPageLinks(
|
||||
slug, fullContent, parsed.frontmatter, parsed.type, resolver,
|
||||
{ globalBasename },
|
||||
{ globalBasename, pack },
|
||||
);
|
||||
|
||||
// Resolve which targets exist (skip refs to non-existent pages to avoid FK
|
||||
|
||||
@@ -116,3 +116,58 @@ export function frontmatterLinkTypeFromPack(
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// #3190 — pack-aware plain link extraction.
|
||||
//
|
||||
// The codegen'd base packs are generated FROM the in-code tables
|
||||
// (inferLinkType's production regexes + FRONTMATTER_LINK_MAP + the
|
||||
// DIR_PATTERN dirs — see scripts/generate-gbrain-base.ts). Consulting them
|
||||
// at extraction time would double-apply WORSE copies of the same semantics:
|
||||
// the YAML carries simplified sketch regexes, and its frontmatter_links
|
||||
// entries have no direction field (the hardcoded map's `incoming` entries
|
||||
// like key_people would come back reversed). Custom packs are the ones
|
||||
// extraction must honor.
|
||||
const CODEGEN_BASE_PACKS = new Set(['gbrain-base', 'gbrain-base-v2']);
|
||||
|
||||
/**
|
||||
* The slice of a resolved pack that plain link extraction
|
||||
* (`extractPageLinks` / `extractFrontmatterLinks`) consumes.
|
||||
*/
|
||||
export interface PackLinkExtraction {
|
||||
link_types: SchemaPackManifest['link_types'];
|
||||
frontmatter_links: SchemaPackManifest['frontmatter_links'];
|
||||
/**
|
||||
* Entity-dir alternatives derived from `page_types[].path_prefixes`
|
||||
* (leading/trailing slashes stripped). Unioned into DIR_PATTERN so
|
||||
* markdown/wikilink targets under pack-declared layouts (e.g.
|
||||
* `wiki/people/…`) become extraction candidates.
|
||||
*/
|
||||
entity_dirs: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Project a resolved active pack down to what link extraction needs.
|
||||
* Returns null (extraction stays purely legacy) when there is no pack or
|
||||
* the pack is one of the codegen'd base packs.
|
||||
*/
|
||||
export function packLinkExtractionView(
|
||||
pack: { manifest: SchemaPackManifest } | null | undefined,
|
||||
): PackLinkExtraction | null {
|
||||
const m = pack?.manifest;
|
||||
if (!m || CODEGEN_BASE_PACKS.has(m.name)) return null;
|
||||
const dirs = new Set<string>();
|
||||
for (const pt of m.page_types) {
|
||||
for (const prefix of pt.path_prefixes ?? []) {
|
||||
const d = prefix.replace(/^\/+/, '').replace(/\/+$/, '');
|
||||
// Only slug-shaped prefixes participate in regex union; anything
|
||||
// else (globs, regex metachars) is skipped rather than escaped so
|
||||
// the entity regexes stay predictable.
|
||||
if (d && /^[A-Za-z0-9][A-Za-z0-9/_-]*$/.test(d)) dirs.add(d);
|
||||
}
|
||||
}
|
||||
return {
|
||||
link_types: m.link_types,
|
||||
frontmatter_links: m.frontmatter_links,
|
||||
entity_dirs: [...dirs],
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
/**
|
||||
* #3190 — pack-aware plain link extraction.
|
||||
*
|
||||
* A custom schema pack's `page_types[].path_prefixes`, `link_types[].inference`
|
||||
* and `frontmatter_links` must drive `extractPageLinks` /
|
||||
* `extractFrontmatterLinks` (the plain fs/db extract path + put_page
|
||||
* auto-link), not just the `--ner` path. Three gates pinned here:
|
||||
* 1. entity-dir whitelist unions in pack path_prefixes,
|
||||
* 2. pack verb regexes are consulted before legacy inferLinkType,
|
||||
* 3. pack frontmatter_links extend FRONTMATTER_LINK_MAP (hardcoded map wins
|
||||
* on collision so base-pack-shaped entries can't emit reversed edges).
|
||||
* Plus the guard: codegen'd base packs are ignored (their semantics already
|
||||
* live in the in-code tables).
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import {
|
||||
extractEntityRefs,
|
||||
extractPageLinks,
|
||||
extractFrontmatterLinks,
|
||||
type SlugResolver,
|
||||
} from '../src/core/link-extraction.ts';
|
||||
import {
|
||||
packLinkExtractionView,
|
||||
type PackLinkExtraction,
|
||||
} from '../src/core/schema-pack/link-inference.ts';
|
||||
import type { PageType } from '../src/core/types.ts';
|
||||
|
||||
const nullResolver: SlugResolver = { resolve: async () => null };
|
||||
|
||||
/** Minimal custom-pack view matching the issue's genealogy repro. */
|
||||
const packView: PackLinkExtraction = {
|
||||
link_types: [
|
||||
{ name: 'parent_of', inference: { regex: '[Pp]arent of \\[' } },
|
||||
{ name: 'member_of_line', inference: { regex: 'member of the \\[' } },
|
||||
],
|
||||
frontmatter_links: [
|
||||
{ page_type: 'person', fields: ['parents'], link_type: 'parent_of' },
|
||||
// Collides with the hardcoded map (company/key_people is incoming
|
||||
// works_at) — must be skipped, not applied outgoing.
|
||||
{ page_type: 'company', fields: ['key_people'], link_type: 'employs' },
|
||||
],
|
||||
entity_dirs: ['wiki/people', 'lines'],
|
||||
};
|
||||
|
||||
describe('#3190 gate 1 — pack path_prefixes widen the entity-dir whitelist', () => {
|
||||
const content = 'member of the [Pettit](../lines/pettit.md) line and ' +
|
||||
'parent of [Harriet](wiki/people/pettit-harriet-emeline-1828.md).';
|
||||
|
||||
test('without extra dirs, non-whitelisted targets yield zero refs', () => {
|
||||
expect(extractEntityRefs(content)).toEqual([]);
|
||||
});
|
||||
|
||||
test('with pack entity dirs, both targets become candidates', () => {
|
||||
const refs = extractEntityRefs(content, packView.entity_dirs);
|
||||
expect(refs.map(r => r.slug).sort()).toEqual([
|
||||
'lines/pettit',
|
||||
'wiki/people/pettit-harriet-emeline-1828',
|
||||
]);
|
||||
});
|
||||
|
||||
test('wikilinks honor pack dirs too', () => {
|
||||
const refs = extractEntityRefs('[[wiki/people/pettit-james-b-1777|James]]', packView.entity_dirs);
|
||||
expect(refs).toHaveLength(1);
|
||||
expect(refs[0].slug).toBe('wiki/people/pettit-james-b-1777');
|
||||
});
|
||||
});
|
||||
|
||||
describe('#3190 gate 2 — pack verb inference runs on the plain extract path', () => {
|
||||
test('pack regex verb wins over legacy mentions', async () => {
|
||||
const { candidates } = await extractPageLinks(
|
||||
'wiki/people/pettit-james-b-1777',
|
||||
'parent of [Harriet](wiki/people/pettit-harriet-emeline-1828.md)',
|
||||
{},
|
||||
'person' as PageType,
|
||||
nullResolver,
|
||||
{ pack: packView },
|
||||
);
|
||||
expect(candidates).toHaveLength(1);
|
||||
expect(candidates[0].targetSlug).toBe('wiki/people/pettit-harriet-emeline-1828');
|
||||
expect(candidates[0].linkType).toBe('parent_of');
|
||||
});
|
||||
|
||||
test('no pack verb match falls through to legacy inference', async () => {
|
||||
const { candidates } = await extractPageLinks(
|
||||
'wiki/people/a',
|
||||
'see also [Someone](wiki/people/someone)',
|
||||
{},
|
||||
'person' as PageType,
|
||||
nullResolver,
|
||||
{ pack: packView },
|
||||
);
|
||||
expect(candidates).toHaveLength(1);
|
||||
expect(candidates[0].linkType).toBe('mentions');
|
||||
});
|
||||
|
||||
test('no pack → legacy behavior unchanged (candidate never exists)', async () => {
|
||||
const { candidates } = await extractPageLinks(
|
||||
'wiki/people/pettit-james-b-1777',
|
||||
'parent of [Harriet](wiki/people/pettit-harriet-emeline-1828.md)',
|
||||
{},
|
||||
'person' as PageType,
|
||||
nullResolver,
|
||||
{},
|
||||
);
|
||||
expect(candidates).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('#3190 gate 3 — pack frontmatter_links extend FRONTMATTER_LINK_MAP', () => {
|
||||
const resolver: SlugResolver = {
|
||||
async resolve(name: string, dirHint?: string | string[]) {
|
||||
const hints = Array.isArray(dirHint) ? dirHint : (dirHint ? [dirHint] : []);
|
||||
// Simulate step-2 resolution: dir-hint + already-slugified value.
|
||||
if (hints.includes('wiki/people')) return `wiki/people/${name}`;
|
||||
return null;
|
||||
},
|
||||
};
|
||||
|
||||
test('pack-only field produces an outgoing typed edge', async () => {
|
||||
const { candidates, unresolved } = await extractFrontmatterLinks(
|
||||
'wiki/people/pettit-harriet-emeline-1828',
|
||||
'person' as PageType,
|
||||
{ parents: ['pettit-james-b-1777', 'felt-lucy-w-1777'] },
|
||||
resolver,
|
||||
packView,
|
||||
);
|
||||
expect(unresolved).toEqual([]);
|
||||
expect(candidates).toHaveLength(2);
|
||||
for (const c of candidates) {
|
||||
expect(c.linkType).toBe('parent_of');
|
||||
expect(c.fromSlug).toBe('wiki/people/pettit-harriet-emeline-1828');
|
||||
expect(c.linkSource).toBe('frontmatter');
|
||||
}
|
||||
expect(candidates.map(c => c.targetSlug).sort()).toEqual([
|
||||
'wiki/people/felt-lucy-w-1777',
|
||||
'wiki/people/pettit-james-b-1777',
|
||||
]);
|
||||
});
|
||||
|
||||
test('hardcoded map wins on colliding (pageType, field) — no reversed duplicate', async () => {
|
||||
const trackingResolver: SlugResolver = {
|
||||
async resolve(name: string) { return `people/${name}`; },
|
||||
};
|
||||
const { candidates } = await extractFrontmatterLinks(
|
||||
'companies/acme',
|
||||
'company' as PageType,
|
||||
{ key_people: ['alice-example'] },
|
||||
trackingResolver,
|
||||
packView,
|
||||
);
|
||||
// Exactly one edge, from the hardcoded incoming works_at mapping —
|
||||
// the pack's outgoing 'employs' restatement is skipped.
|
||||
expect(candidates).toHaveLength(1);
|
||||
expect(candidates[0].linkType).toBe('works_at');
|
||||
expect(candidates[0].fromSlug).toBe('people/alice-example');
|
||||
expect(candidates[0].targetSlug).toBe('companies/acme');
|
||||
});
|
||||
});
|
||||
|
||||
describe('#3190 — packLinkExtractionView', () => {
|
||||
const manifest = (over: Record<string, unknown>) => ({
|
||||
manifest: {
|
||||
name: 'family-pack',
|
||||
page_types: [
|
||||
{ name: 'person', path_prefixes: ['wiki/people/', '/lines/'] },
|
||||
{ name: 'weird', path_prefixes: ['../evil/', 'ok_dir/'] },
|
||||
],
|
||||
link_types: [],
|
||||
frontmatter_links: [],
|
||||
...over,
|
||||
} as never,
|
||||
});
|
||||
|
||||
test('derives slug-safe entity dirs from path_prefixes', () => {
|
||||
const view = packLinkExtractionView(manifest({}));
|
||||
expect(view).not.toBeNull();
|
||||
expect(view!.entity_dirs.sort()).toEqual(['lines', 'ok_dir', 'wiki/people']);
|
||||
});
|
||||
|
||||
test('returns null for codegen base packs and missing packs', () => {
|
||||
expect(packLinkExtractionView(null)).toBeNull();
|
||||
expect(packLinkExtractionView(manifest({ name: 'gbrain-base' }))).toBeNull();
|
||||
expect(packLinkExtractionView(manifest({ name: 'gbrain-base-v2' }))).toBeNull();
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user