From 78f3ebea339ce2c38111d73ea01f0476678ebefc Mon Sep 17 00:00:00 2001 From: Paolo Belcastro Date: Sun, 5 Jul 2026 17:20:15 +0200 Subject: [PATCH] feat(extract_atoms): honor pack manifest extractable flag in page discovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Atom-extraction page discovery hardcoded EXTRACTABLE_PAGE_TYPES and ignored the active pack's `extractable: true` flags, so a type declared extractable in the manifest (e.g. `note`) never actually extracted. Closes the D2 TODO the code already flagged (extract-atoms.ts: 'future pack-aware refactor ... pull from the active pack manifest'). Resolve the allowlist as: legacy hardcoded floor UNION the pack's extractable types, MINUS synthesis outputs (atom, concept — extracting from these would loop, since concepts are synthesized from atoms). Mirrors facts/eligibility.ts, which excludes concept the same way. Back-compat: gbrain-base brains keep every legacy target via the union; fail-soft falls back to the legacy floor if the pack can't load. - pure unionExtractableTypes() policy, unit-tested - discoverExtractablePages + countExtractAtomsBacklog resolve from the pack - page-discovery fixture updated (note now extracts; concept stays excluded) Co-Authored-By: Claude Opus 4.8 --- src/core/cycle/extract-atoms.ts | 57 +++++++++++++++++--- test/extract-atoms-extractable-types.test.ts | 39 ++++++++++++++ test/extract-atoms-page-discovery.test.ts | 12 +++-- 3 files changed, 96 insertions(+), 12 deletions(-) create mode 100644 test/extract-atoms-extractable-types.test.ts diff --git a/src/core/cycle/extract-atoms.ts b/src/core/cycle/extract-atoms.ts index f1e94bb06..582a41044 100644 --- a/src/core/cycle/extract-atoms.ts +++ b/src/core/cycle/extract-atoms.ts @@ -61,16 +61,56 @@ const ATOM_TYPES = [ 'critique', 'collection', ] as const; -// v0.41.2.1 (D2): brain-page discovery constants. Hardcoded for now; -// future pack-aware refactor is a one-line change to pull from the -// active pack manifest (symmetric with the existing -// src/core/facts/eligibility.ts:49 TODO). -const EXTRACTABLE_PAGE_TYPES = [ +// v0.41.2.1 (D2): brain-page discovery constants. +// +// Legacy floor: the pre-pack hardcoded atom-extraction types. Retained as a +// back-compat union member so a gbrain-base brain never loses an extraction +// target when we begin honoring the pack manifest's `extractable` flags. +const LEGACY_EXTRACTABLE_TYPES = [ 'meeting', 'source', 'article', 'video', 'book', 'original', ] as const; + +// Synthesis outputs are never extraction inputs: extracting atoms from atoms or +// concepts would loop (concepts are synthesized FROM atoms). Mirrors +// facts/eligibility.ts, which likewise excludes `concept` despite its +// extractable:true flag being a documented forward-compat marker. +const SYNTHESIS_OUTPUT_TYPES = new Set(['atom', 'concept']); + const PAGE_DISCOVERY_BUDGET = 50; const MIN_PAGE_CHARS_FOR_EXTRACTION = 500; +/** + * Pure allowlist policy: the legacy floor UNION the pack's `extractable: true` + * types, MINUS synthesis outputs. Exported for unit tests; keep I/O-free. + */ +export function unionExtractableTypes(packExtractable: Iterable): string[] { + const types = new Set(LEGACY_EXTRACTABLE_TYPES); + for (const t of packExtractable) types.add(t); + for (const t of SYNTHESIS_OUTPUT_TYPES) types.delete(t); + return [...types]; +} + +/** + * Resolve the atom-extraction type allowlist from the active schema pack. + * Closes the D2 TODO of honoring the pack manifest (so a type declared + * extractable — e.g. `note` — actually extracts) while preserving behavior for + * gbrain-base via the legacy-floor union. Fail-soft: any pack-load error falls + * back to the legacy floor. + */ +async function resolveExtractableTypes(): Promise { + let packExtractable: Iterable = []; + try { + const { loadConfig } = await import('../config.ts'); + const { loadActivePack } = await import('../schema-pack/load-active.ts'); + const { extractableTypesFromPack } = await import('../schema-pack/extractable.ts'); + const resolved = await loadActivePack({ cfg: loadConfig(), remote: false }); + packExtractable = extractableTypesFromPack(resolved.manifest); + } catch { + // Pack unavailable (test seams, bootstrap) — legacy floor only. + } + return unionExtractableTypes(packExtractable); +} + export interface ExtractAtomsOpts { brainDir?: string; sourceId?: string; @@ -195,7 +235,7 @@ export async function discoverExtractablePages( `; const params: unknown[] = [ sourceId, - EXTRACTABLE_PAGE_TYPES as unknown as string[], + await resolveExtractableTypes(), MIN_PAGE_CHARS_FOR_EXTRACTION, PAGE_DISCOVERY_BUDGET, ]; @@ -272,9 +312,10 @@ export async function countExtractAtomsBacklog( AND atom.frontmatter->>'source_hash' = substring(p.content_hash from 1 for 16) AND atom.deleted_at IS NULL )`; + const extractableTypes = await resolveExtractableTypes(); const params = scoped - ? [sourceId, EXTRACTABLE_PAGE_TYPES as unknown as string[], MIN_PAGE_CHARS_FOR_EXTRACTION] - : [EXTRACTABLE_PAGE_TYPES as unknown as string[], MIN_PAGE_CHARS_FOR_EXTRACTION]; + ? [sourceId, extractableTypes, MIN_PAGE_CHARS_FOR_EXTRACTION] + : [extractableTypes, MIN_PAGE_CHARS_FOR_EXTRACTION]; const rows = await engine.executeRaw<{ cnt: string | number }>(sql, params); return Number(rows[0]?.cnt ?? 0); } catch (err) { diff --git a/test/extract-atoms-extractable-types.test.ts b/test/extract-atoms-extractable-types.test.ts new file mode 100644 index 000000000..4dacb2ccb --- /dev/null +++ b/test/extract-atoms-extractable-types.test.ts @@ -0,0 +1,39 @@ +/** + * Pack-driven extractable-type allowlist (unionExtractableTypes): honors the + * schema-pack manifest's `extractable: true` flags while preserving the legacy + * hardcoded floor and excluding synthesis outputs. Closes the D2 TODO in + * extract-atoms.ts (page discovery was ignoring the pack's extractable flag, so + * a type declared extractable — e.g. `note` — never actually extracted). + */ +import { describe, test, expect } from 'bun:test'; +import { unionExtractableTypes } from '../src/core/cycle/extract-atoms.ts'; + +const LEGACY = ['meeting', 'source', 'article', 'video', 'book', 'original']; + +describe('unionExtractableTypes', () => { + test('legacy floor is always present (back-compat)', () => { + const r = unionExtractableTypes([]); + for (const t of LEGACY) expect(r).toContain(t); + }); + + test('pack-declared extractable types are added (e.g. note)', () => { + const r = unionExtractableTypes(['note', 'writing']); + expect(r).toContain('note'); + expect(r).toContain('writing'); + for (const t of LEGACY) expect(r).toContain(t); + }); + + test('synthesis outputs are excluded even when the pack marks them extractable', () => { + // gbrain-base declares `concept` extractable:true, but extracting atoms FROM + // concepts would loop (concepts are synthesized from atoms). + const r = unionExtractableTypes(['note', 'concept', 'atom']); + expect(r).toContain('note'); + expect(r).not.toContain('concept'); + expect(r).not.toContain('atom'); + }); + + test('no duplicates when the pack repeats a legacy type', () => { + const r = unionExtractableTypes(['meeting', 'source']); + expect(r.filter((t) => t === 'meeting')).toHaveLength(1); + }); +}); diff --git a/test/extract-atoms-page-discovery.test.ts b/test/extract-atoms-page-discovery.test.ts index db459c64a..50919b21e 100644 --- a/test/extract-atoms-page-discovery.test.ts +++ b/test/extract-atoms-page-discovery.test.ts @@ -108,12 +108,15 @@ async function seedPage(opts: { } describe('v0.41.2.1: discoverExtractablePages SQL contract', () => { - test('filters by all 6 extractable types', async () => { - for (const type of ['meeting', 'source', 'article', 'video', 'book', 'original']) { + test('discovers legacy + pack-extractable types, excludes synthesis outputs', async () => { + // Legacy floor + `note` (declared extractable:true in gbrain-base, now + // honored via the pack manifest — the D2 fix). + for (const type of ['meeting', 'source', 'article', 'video', 'book', 'original', 'note']) { await seedPage({ slug: `${type}/x`, type }); } - // Add a non-extractable page that should NOT appear - await seedPage({ slug: 'notes/skip-me', type: 'note' }); + // `concept` is also extractable:true in gbrain-base, but extracting atoms + // FROM concepts would loop — synthesis outputs are always excluded. + await seedPage({ slug: 'wiki/concepts/skip-me', type: 'concept' }); const discovered = await discoverExtractablePages(engine, 'default'); const slugs = discovered.map((d) => d.slug).sort(); @@ -121,6 +124,7 @@ describe('v0.41.2.1: discoverExtractablePages SQL contract', () => { 'article/x', 'book/x', 'meeting/x', + 'note/x', 'original/x', 'source/x', 'video/x',