diff --git a/src/cli.ts b/src/cli.ts index bee3da91c..6bb70651e 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -18,7 +18,7 @@ for (const op of operations) { } // CLI-only commands that bypass the operation layer -const CLI_ONLY = new Set(['init', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'features', 'autopilot', 'graph-query', 'jobs', 'apply-migrations', 'skillpack-check']); +const CLI_ONLY = new Set(['init', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'features', 'autopilot', 'graph-query', 'jobs', 'apply-migrations', 'skillpack-check', 'orphans']); async function main() { const args = process.argv.slice(2); @@ -412,6 +412,11 @@ async function handleCliOnly(command: string, args: string[]) { await runGraphQuery(engine, args); break; } + case 'orphans': { + const { runOrphans } = await import('./commands/orphans.ts'); + await runOrphans(engine, args); + break; + } } } finally { if (command !== 'serve') await engine.disconnect(); @@ -520,6 +525,7 @@ TOOLS publish [--password] Shareable HTML (strips private data, optional AES-256) check-backlinks [dir] Find/fix missing back-links across brain lint [--fix] Catch LLM artifacts, placeholder dates, bad frontmatter + orphans [--json] [--count] Find pages with no inbound wikilinks report --type --content ... Save timestamped report to brain/reports/ JOBS (Minions) diff --git a/src/commands/extract.ts b/src/commands/extract.ts index 87f50330d..11157b93c 100644 --- a/src/commands/extract.ts +++ b/src/commands/extract.ts @@ -156,19 +156,7 @@ export function resolveSlug(fileDir: string, relTarget: string, allSlugs: Set): string { - const from = fromDir.split('/')[0]; - const to = toDir.split('/')[0]; - if (from === 'people' && to === 'companies') { - if (Array.isArray(frontmatter?.founded)) return 'founded'; - return 'works_at'; - } - if (from === 'people' && to === 'deals') return 'involved_in'; - if (from === 'deals' && to === 'companies') return 'deal_for'; - if (from === 'meetings' && to === 'people') return 'attendee'; - return 'mention'; -} +// inferLinkType is now imported from ../core/link-extraction.ts (v0.12.0 canonical extractor) /** Extract links from frontmatter fields */ function extractFrontmatterLinks(slug: string, fm: Record): ExtractedLink[] { diff --git a/src/commands/orphans.ts b/src/commands/orphans.ts new file mode 100644 index 000000000..43413f3c1 --- /dev/null +++ b/src/commands/orphans.ts @@ -0,0 +1,227 @@ +/** + * gbrain orphans — Surface pages with no inbound wikilinks. + * + * Deterministic: zero LLM calls. Queries the links table for pages with + * no entries where to_page_id = pages.id. By default filters out + * auto-generated pages and pseudo-pages where no inbound links is expected. + * + * Usage: + * gbrain orphans # list orphans grouped by domain + * gbrain orphans --json # JSON output for agent consumption + * gbrain orphans --count # just the number + * gbrain orphans --include-pseudo # include auto-generated/pseudo pages + */ + +import type { BrainEngine } from '../core/engine.ts'; +import * as db from '../core/db.ts'; + +// --- Types --- + +export interface OrphanPage { + slug: string; + title: string; + domain: string; +} + +export interface OrphanResult { + orphans: OrphanPage[]; + total_orphans: number; + total_linkable: number; + total_pages: number; + excluded: number; +} + +// --- Filter constants --- + +/** Slug suffixes that are always auto-generated root files */ +const AUTO_SUFFIX_PATTERNS = ['/_index', '/log']; + +/** Page slugs that are pseudo-pages by convention */ +const PSEUDO_SLUGS = new Set(['_atlas', '_index', '_stats', '_orphans', '_scratch', 'claude']); + +/** Slug segment that marks raw sources */ +const RAW_SEGMENT = '/raw/'; + +/** Slug prefixes where no inbound links is expected */ +const DENY_PREFIXES = [ + 'output/', + 'dashboards/', + 'scripts/', + 'templates/', + 'openclaw/config/', +]; + +/** First slug segments where no inbound links is expected */ +const FIRST_SEGMENT_EXCLUSIONS = new Set(['scratch', 'thoughts', 'catalog', 'entities']); + +// --- Filter logic --- + +/** + * Returns true if a slug should be excluded from orphan reporting by default. + * These are pages where having no inbound links is expected / not a content problem. + */ +export function shouldExclude(slug: string): boolean { + // Pseudo-pages (exact match) + if (PSEUDO_SLUGS.has(slug)) return true; + + // Auto-generated suffix patterns + for (const suffix of AUTO_SUFFIX_PATTERNS) { + if (slug.endsWith(suffix)) return true; + } + + // Raw source slugs + if (slug.includes(RAW_SEGMENT)) return true; + + // Deny-prefix slugs + for (const prefix of DENY_PREFIXES) { + if (slug.startsWith(prefix)) return true; + } + + // First-segment exclusions + const firstSegment = slug.split('/')[0]; + if (FIRST_SEGMENT_EXCLUSIONS.has(firstSegment)) return true; + + return false; +} + +/** + * Derive domain from frontmatter or first slug segment. + */ +export function deriveDomain(frontmatterDomain: string | null | undefined, slug: string): string { + if (frontmatterDomain && typeof frontmatterDomain === 'string' && frontmatterDomain.trim()) { + return frontmatterDomain.trim(); + } + return slug.split('/')[0] || 'root'; +} + +// --- Core query --- + +/** + * Find pages with no inbound links. + * Returns raw rows from the DB (all pages regardless of filter). + */ +export async function queryOrphanPages(): Promise<{ slug: string; title: string; domain: string | null }[]> { + const sql = db.getConnection(); + const rows = await sql` + SELECT + p.slug, + COALESCE(p.title, p.slug) AS title, + p.frontmatter->>'domain' AS domain + FROM pages p + WHERE NOT EXISTS ( + SELECT 1 FROM links l WHERE l.to_page_id = p.id + ) + ORDER BY p.slug + `; + return rows as { slug: string; title: string; domain: string | null }[]; +} + +/** + * Find orphan pages, with optional pseudo-page filtering. + * Returns structured OrphanResult with totals. + */ +export async function findOrphans(includePseudo: boolean = false): Promise { + const allOrphans = await queryOrphanPages(); + const totalPages = allOrphans.length; // pages with no inbound links + + // Count total pages in DB for the summary line + const sql = db.getConnection(); + const [{ count: totalPagesCount }] = await sql`SELECT count(*)::int AS count FROM pages`; + const total = Number(totalPagesCount); + + const filtered = includePseudo + ? allOrphans + : allOrphans.filter(row => !shouldExclude(row.slug)); + + const orphans: OrphanPage[] = filtered.map(row => ({ + slug: row.slug, + title: row.title, + domain: deriveDomain(row.domain, row.slug), + })); + + const excluded = allOrphans.length - filtered.length; + + return { + orphans, + total_orphans: orphans.length, + total_linkable: filtered.length + (total - allOrphans.length), + total_pages: total, + excluded, + }; +} + +// --- Output formatters --- + +export function formatOrphansText(result: OrphanResult): string { + const lines: string[] = []; + + const { orphans, total_orphans, total_linkable, total_pages, excluded } = result; + lines.push( + `${total_orphans} orphans out of ${total_linkable} linkable pages (${total_pages} total; ${excluded} excluded)\n`, + ); + + if (orphans.length === 0) { + lines.push('No orphan pages found.'); + return lines.join('\n'); + } + + // Group by domain, sort alphabetically within each group + const byDomain = new Map(); + for (const page of orphans) { + const list = byDomain.get(page.domain) || []; + list.push(page); + byDomain.set(page.domain, list); + } + + // Sort domains alphabetically + const sortedDomains = [...byDomain.keys()].sort(); + for (const domain of sortedDomains) { + const pages = byDomain.get(domain)!.sort((a, b) => a.slug.localeCompare(b.slug)); + lines.push(`[${domain}]`); + for (const page of pages) { + lines.push(` ${page.slug} ${page.title}`); + } + lines.push(''); + } + + return lines.join('\n').trimEnd(); +} + +// --- CLI entry point --- + +export async function runOrphans(_engine: BrainEngine, args: string[]) { + const json = args.includes('--json'); + const count = args.includes('--count'); + const includePseudo = args.includes('--include-pseudo'); + + if (args.includes('--help') || args.includes('-h')) { + console.log(`Usage: gbrain orphans [options] + +Find pages with no inbound wikilinks. + +Options: + --json Output as JSON (for agent consumption) + --count Output just the number of orphans + --include-pseudo Include auto-generated and pseudo pages in results + --help, -h Show this help + +Output (default): grouped by domain, sorted alphabetically within each group +Summary line: N orphans out of M linkable pages (K total; K-M excluded) +`); + return; + } + + const result = await findOrphans(includePseudo); + + if (count) { + console.log(String(result.total_orphans)); + return; + } + + if (json) { + console.log(JSON.stringify(result, null, 2)); + return; + } + + console.log(formatOrphansText(result)); +} diff --git a/src/core/operations.ts b/src/core/operations.ts index 2f266cbe5..ec6e41218 100644 --- a/src/core/operations.ts +++ b/src/core/operations.ts @@ -1082,6 +1082,24 @@ const send_job_message: Operation = { }, }; +// --- Orphans --- + +const find_orphans: Operation = { + name: 'find_orphans', + description: 'Find pages with no inbound wikilinks. Essential for content enrichment cycles.', + params: { + include_pseudo: { + type: 'boolean', + description: 'Include auto-generated and pseudo pages (default: false)', + }, + }, + handler: async (_ctx, p) => { + const { findOrphans } = await import('../commands/orphans.ts'); + return findOrphans((p.include_pseudo as boolean) || false); + }, + cliHints: { name: 'orphans', hidden: true }, +}; + // --- Exports --- export const operations: Operation[] = [ @@ -1110,6 +1128,8 @@ export const operations: Operation[] = [ // Jobs (Minions) submit_job, get_job, list_jobs, cancel_job, retry_job, get_job_progress, pause_job, resume_job, replay_job, send_job_message, + // Orphans + find_orphans, ]; export const operationsByName = Object.fromEntries( diff --git a/test/orphans.test.ts b/test/orphans.test.ts new file mode 100644 index 000000000..d6748830d --- /dev/null +++ b/test/orphans.test.ts @@ -0,0 +1,203 @@ +import { describe, test, expect } from 'bun:test'; +import { + shouldExclude, + deriveDomain, + formatOrphansText, + type OrphanPage, + type OrphanResult, +} from '../src/commands/orphans.ts'; + +// --- shouldExclude --- + +describe('shouldExclude', () => { + test('excludes pseudo-page _atlas', () => { + expect(shouldExclude('_atlas')).toBe(true); + }); + + test('excludes pseudo-page _index', () => { + expect(shouldExclude('_index')).toBe(true); + }); + + test('excludes pseudo-page _stats', () => { + expect(shouldExclude('_stats')).toBe(true); + }); + + test('excludes pseudo-page _orphans', () => { + expect(shouldExclude('_orphans')).toBe(true); + }); + + test('excludes pseudo-page _scratch', () => { + expect(shouldExclude('_scratch')).toBe(true); + }); + + test('excludes pseudo-page claude', () => { + expect(shouldExclude('claude')).toBe(true); + }); + + test('excludes auto-generated _index suffix', () => { + expect(shouldExclude('companies/_index')).toBe(true); + expect(shouldExclude('people/_index')).toBe(true); + }); + + test('excludes auto-generated /log suffix', () => { + expect(shouldExclude('projects/acme/log')).toBe(true); + }); + + test('excludes raw source slugs', () => { + expect(shouldExclude('companies/acme/raw/crustdata')).toBe(true); + }); + + test('excludes deny-prefix: output/', () => { + expect(shouldExclude('output/2026-q1')).toBe(true); + }); + + test('excludes deny-prefix: dashboards/', () => { + expect(shouldExclude('dashboards/metrics')).toBe(true); + }); + + test('excludes deny-prefix: scripts/', () => { + expect(shouldExclude('scripts/ingest-runner')).toBe(true); + }); + + test('excludes deny-prefix: templates/', () => { + expect(shouldExclude('templates/meeting-note')).toBe(true); + }); + + test('excludes deny-prefix: openclaw/config/', () => { + expect(shouldExclude('openclaw/config/agent')).toBe(true); + }); + + test('excludes first-segment: scratch', () => { + expect(shouldExclude('scratch/idea-dump')).toBe(true); + }); + + test('excludes first-segment: thoughts', () => { + expect(shouldExclude('thoughts/2026-04-17')).toBe(true); + }); + + test('excludes first-segment: catalog', () => { + expect(shouldExclude('catalog/tools')).toBe(true); + }); + + test('excludes first-segment: entities', () => { + expect(shouldExclude('entities/product-hunt')).toBe(true); + }); + + test('does NOT exclude a normal content page', () => { + expect(shouldExclude('companies/acme')).toBe(false); + expect(shouldExclude('people/jane-doe')).toBe(false); + expect(shouldExclude('projects/gbrain')).toBe(false); + }); + + test('does NOT exclude a page ending with log-like text that is not /log', () => { + expect(shouldExclude('devlog')).toBe(false); + expect(shouldExclude('changelog')).toBe(false); + }); +}); + +// --- deriveDomain --- + +describe('deriveDomain', () => { + test('uses frontmatter domain when present', () => { + expect(deriveDomain('companies', 'companies/acme')).toBe('companies'); + }); + + test('falls back to first slug segment', () => { + expect(deriveDomain(null, 'people/jane-doe')).toBe('people'); + expect(deriveDomain(undefined, 'projects/gbrain')).toBe('projects'); + }); + + test('returns root for single-segment slugs with no frontmatter', () => { + expect(deriveDomain(null, 'readme')).toBe('readme'); + }); + + test('ignores empty-string frontmatter domain', () => { + expect(deriveDomain('', 'people/alice')).toBe('people'); + }); + + test('ignores whitespace-only frontmatter domain', () => { + expect(deriveDomain(' ', 'people/alice')).toBe('people'); + }); +}); + +// --- formatOrphansText --- + +describe('formatOrphansText', () => { + function makeResult(orphans: OrphanPage[], overrides?: Partial): OrphanResult { + return { + orphans, + total_orphans: orphans.length, + total_linkable: orphans.length + 50, + total_pages: orphans.length + 60, + excluded: 10, + ...overrides, + }; + } + + test('shows summary line', () => { + const result = makeResult([]); + const out = formatOrphansText(result); + expect(out).toContain('0 orphans out of'); + expect(out).toContain('total'); + expect(out).toContain('excluded'); + }); + + test('shows "No orphan pages found." when empty', () => { + const out = formatOrphansText(makeResult([])); + expect(out).toContain('No orphan pages found.'); + }); + + test('groups orphans by domain', () => { + const orphans: OrphanPage[] = [ + { slug: 'companies/acme', title: 'Acme Corp', domain: 'companies' }, + { slug: 'people/alice', title: 'Alice', domain: 'people' }, + { slug: 'companies/beta', title: 'Beta Inc', domain: 'companies' }, + ]; + const out = formatOrphansText(makeResult(orphans)); + expect(out).toContain('[companies]'); + expect(out).toContain('[people]'); + // companies section should appear before people (alphabetical) + const companiesIdx = out.indexOf('[companies]'); + const peopleIdx = out.indexOf('[people]'); + expect(companiesIdx).toBeLessThan(peopleIdx); + }); + + test('sorts orphans alphabetically within each domain group', () => { + const orphans: OrphanPage[] = [ + { slug: 'companies/zeta', title: 'Zeta', domain: 'companies' }, + { slug: 'companies/alpha', title: 'Alpha', domain: 'companies' }, + { slug: 'companies/beta', title: 'Beta', domain: 'companies' }, + ]; + const out = formatOrphansText(makeResult(orphans)); + const alphaIdx = out.indexOf('companies/alpha'); + const betaIdx = out.indexOf('companies/beta'); + const zetaIdx = out.indexOf('companies/zeta'); + expect(alphaIdx).toBeLessThan(betaIdx); + expect(betaIdx).toBeLessThan(zetaIdx); + }); + + test('includes slug and title in output', () => { + const orphans: OrphanPage[] = [ + { slug: 'companies/acme', title: 'Acme Corp', domain: 'companies' }, + ]; + const out = formatOrphansText(makeResult(orphans)); + expect(out).toContain('companies/acme'); + expect(out).toContain('Acme Corp'); + }); + + test('summary line shows correct numbers', () => { + const orphans: OrphanPage[] = [ + { slug: 'a/b', title: 'B', domain: 'a' }, + { slug: 'a/c', title: 'C', domain: 'a' }, + ]; + const result: OrphanResult = { + orphans, + total_orphans: 2, + total_linkable: 100, + total_pages: 120, + excluded: 20, + }; + const out = formatOrphansText(result); + expect(out).toContain('2 orphans out of 100 linkable pages (120 total; 20 excluded)'); + }); +});