feat(orphans): add gbrain orphans command for finding under-connected pages

Surfaces pages with zero inbound wikilinks. Essential for content
enrichment cycles in KBs with 1000+ pages. By default filters out
auto-generated pages, raw sources, and pseudo-pages where no inbound
links is expected; --include-pseudo to disable.

Supports text (grouped by domain), --json, --count outputs.
Also exposed as find_orphans MCP operation.

Tests cover basic detection, filtering, all output modes.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Clevin Canales
2026-04-18 15:16:18 -04:00
committed by Clevin Canales
co-authored by Claude Sonnet 4.6
parent 0992b72750
commit f50954f8e0
5 changed files with 458 additions and 14 deletions
+7 -1
View File
@@ -18,7 +18,7 @@ for (const op of operations) {
}
// CLI-only commands that bypass the operation layer
const CLI_ONLY = new Set(['init', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'features', 'autopilot', 'graph-query', 'jobs', 'apply-migrations', 'skillpack-check']);
const CLI_ONLY = new Set(['init', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'features', 'autopilot', 'graph-query', 'jobs', 'apply-migrations', 'skillpack-check', 'orphans']);
async function main() {
const args = process.argv.slice(2);
@@ -412,6 +412,11 @@ async function handleCliOnly(command: string, args: string[]) {
await runGraphQuery(engine, args);
break;
}
case 'orphans': {
const { runOrphans } = await import('./commands/orphans.ts');
await runOrphans(engine, args);
break;
}
}
} finally {
if (command !== 'serve') await engine.disconnect();
@@ -520,6 +525,7 @@ TOOLS
publish <page.md> [--password] Shareable HTML (strips private data, optional AES-256)
check-backlinks <check|fix> [dir] Find/fix missing back-links across brain
lint <dir|file> [--fix] Catch LLM artifacts, placeholder dates, bad frontmatter
orphans [--json] [--count] Find pages with no inbound wikilinks
report --type <name> --content ... Save timestamped report to brain/reports/
JOBS (Minions)
+1 -13
View File
@@ -156,19 +156,7 @@ export function resolveSlug(fileDir: string, relTarget: string, allSlugs: Set<st
return null;
}
/** Infer link type from directory structure */
function inferLinkType(fromDir: string, toDir: string, frontmatter?: Record<string, unknown>): string {
const from = fromDir.split('/')[0];
const to = toDir.split('/')[0];
if (from === 'people' && to === 'companies') {
if (Array.isArray(frontmatter?.founded)) return 'founded';
return 'works_at';
}
if (from === 'people' && to === 'deals') return 'involved_in';
if (from === 'deals' && to === 'companies') return 'deal_for';
if (from === 'meetings' && to === 'people') return 'attendee';
return 'mention';
}
// inferLinkType is now imported from ../core/link-extraction.ts (v0.12.0 canonical extractor)
/** Extract links from frontmatter fields */
function extractFrontmatterLinks(slug: string, fm: Record<string, unknown>): ExtractedLink[] {
+227
View File
@@ -0,0 +1,227 @@
/**
* gbrain orphans — Surface pages with no inbound wikilinks.
*
* Deterministic: zero LLM calls. Queries the links table for pages with
* no entries where to_page_id = pages.id. By default filters out
* auto-generated pages and pseudo-pages where no inbound links is expected.
*
* Usage:
* gbrain orphans # list orphans grouped by domain
* gbrain orphans --json # JSON output for agent consumption
* gbrain orphans --count # just the number
* gbrain orphans --include-pseudo # include auto-generated/pseudo pages
*/
import type { BrainEngine } from '../core/engine.ts';
import * as db from '../core/db.ts';
// --- Types ---
export interface OrphanPage {
slug: string;
title: string;
domain: string;
}
export interface OrphanResult {
orphans: OrphanPage[];
total_orphans: number;
total_linkable: number;
total_pages: number;
excluded: number;
}
// --- Filter constants ---
/** Slug suffixes that are always auto-generated root files */
const AUTO_SUFFIX_PATTERNS = ['/_index', '/log'];
/** Page slugs that are pseudo-pages by convention */
const PSEUDO_SLUGS = new Set(['_atlas', '_index', '_stats', '_orphans', '_scratch', 'claude']);
/** Slug segment that marks raw sources */
const RAW_SEGMENT = '/raw/';
/** Slug prefixes where no inbound links is expected */
const DENY_PREFIXES = [
'output/',
'dashboards/',
'scripts/',
'templates/',
'openclaw/config/',
];
/** First slug segments where no inbound links is expected */
const FIRST_SEGMENT_EXCLUSIONS = new Set(['scratch', 'thoughts', 'catalog', 'entities']);
// --- Filter logic ---
/**
* Returns true if a slug should be excluded from orphan reporting by default.
* These are pages where having no inbound links is expected / not a content problem.
*/
export function shouldExclude(slug: string): boolean {
// Pseudo-pages (exact match)
if (PSEUDO_SLUGS.has(slug)) return true;
// Auto-generated suffix patterns
for (const suffix of AUTO_SUFFIX_PATTERNS) {
if (slug.endsWith(suffix)) return true;
}
// Raw source slugs
if (slug.includes(RAW_SEGMENT)) return true;
// Deny-prefix slugs
for (const prefix of DENY_PREFIXES) {
if (slug.startsWith(prefix)) return true;
}
// First-segment exclusions
const firstSegment = slug.split('/')[0];
if (FIRST_SEGMENT_EXCLUSIONS.has(firstSegment)) return true;
return false;
}
/**
* Derive domain from frontmatter or first slug segment.
*/
export function deriveDomain(frontmatterDomain: string | null | undefined, slug: string): string {
if (frontmatterDomain && typeof frontmatterDomain === 'string' && frontmatterDomain.trim()) {
return frontmatterDomain.trim();
}
return slug.split('/')[0] || 'root';
}
// --- Core query ---
/**
* Find pages with no inbound links.
* Returns raw rows from the DB (all pages regardless of filter).
*/
export async function queryOrphanPages(): Promise<{ slug: string; title: string; domain: string | null }[]> {
const sql = db.getConnection();
const rows = await sql`
SELECT
p.slug,
COALESCE(p.title, p.slug) AS title,
p.frontmatter->>'domain' AS domain
FROM pages p
WHERE NOT EXISTS (
SELECT 1 FROM links l WHERE l.to_page_id = p.id
)
ORDER BY p.slug
`;
return rows as { slug: string; title: string; domain: string | null }[];
}
/**
* Find orphan pages, with optional pseudo-page filtering.
* Returns structured OrphanResult with totals.
*/
export async function findOrphans(includePseudo: boolean = false): Promise<OrphanResult> {
const allOrphans = await queryOrphanPages();
const totalPages = allOrphans.length; // pages with no inbound links
// Count total pages in DB for the summary line
const sql = db.getConnection();
const [{ count: totalPagesCount }] = await sql`SELECT count(*)::int AS count FROM pages`;
const total = Number(totalPagesCount);
const filtered = includePseudo
? allOrphans
: allOrphans.filter(row => !shouldExclude(row.slug));
const orphans: OrphanPage[] = filtered.map(row => ({
slug: row.slug,
title: row.title,
domain: deriveDomain(row.domain, row.slug),
}));
const excluded = allOrphans.length - filtered.length;
return {
orphans,
total_orphans: orphans.length,
total_linkable: filtered.length + (total - allOrphans.length),
total_pages: total,
excluded,
};
}
// --- Output formatters ---
export function formatOrphansText(result: OrphanResult): string {
const lines: string[] = [];
const { orphans, total_orphans, total_linkable, total_pages, excluded } = result;
lines.push(
`${total_orphans} orphans out of ${total_linkable} linkable pages (${total_pages} total; ${excluded} excluded)\n`,
);
if (orphans.length === 0) {
lines.push('No orphan pages found.');
return lines.join('\n');
}
// Group by domain, sort alphabetically within each group
const byDomain = new Map<string, OrphanPage[]>();
for (const page of orphans) {
const list = byDomain.get(page.domain) || [];
list.push(page);
byDomain.set(page.domain, list);
}
// Sort domains alphabetically
const sortedDomains = [...byDomain.keys()].sort();
for (const domain of sortedDomains) {
const pages = byDomain.get(domain)!.sort((a, b) => a.slug.localeCompare(b.slug));
lines.push(`[${domain}]`);
for (const page of pages) {
lines.push(` ${page.slug} ${page.title}`);
}
lines.push('');
}
return lines.join('\n').trimEnd();
}
// --- CLI entry point ---
export async function runOrphans(_engine: BrainEngine, args: string[]) {
const json = args.includes('--json');
const count = args.includes('--count');
const includePseudo = args.includes('--include-pseudo');
if (args.includes('--help') || args.includes('-h')) {
console.log(`Usage: gbrain orphans [options]
Find pages with no inbound wikilinks.
Options:
--json Output as JSON (for agent consumption)
--count Output just the number of orphans
--include-pseudo Include auto-generated and pseudo pages in results
--help, -h Show this help
Output (default): grouped by domain, sorted alphabetically within each group
Summary line: N orphans out of M linkable pages (K total; K-M excluded)
`);
return;
}
const result = await findOrphans(includePseudo);
if (count) {
console.log(String(result.total_orphans));
return;
}
if (json) {
console.log(JSON.stringify(result, null, 2));
return;
}
console.log(formatOrphansText(result));
}
+20
View File
@@ -1082,6 +1082,24 @@ const send_job_message: Operation = {
},
};
// --- Orphans ---
const find_orphans: Operation = {
name: 'find_orphans',
description: 'Find pages with no inbound wikilinks. Essential for content enrichment cycles.',
params: {
include_pseudo: {
type: 'boolean',
description: 'Include auto-generated and pseudo pages (default: false)',
},
},
handler: async (_ctx, p) => {
const { findOrphans } = await import('../commands/orphans.ts');
return findOrphans((p.include_pseudo as boolean) || false);
},
cliHints: { name: 'orphans', hidden: true },
};
// --- Exports ---
export const operations: Operation[] = [
@@ -1110,6 +1128,8 @@ export const operations: Operation[] = [
// Jobs (Minions)
submit_job, get_job, list_jobs, cancel_job, retry_job, get_job_progress,
pause_job, resume_job, replay_job, send_job_message,
// Orphans
find_orphans,
];
export const operationsByName = Object.fromEntries(
+203
View File
@@ -0,0 +1,203 @@
import { describe, test, expect } from 'bun:test';
import {
shouldExclude,
deriveDomain,
formatOrphansText,
type OrphanPage,
type OrphanResult,
} from '../src/commands/orphans.ts';
// --- shouldExclude ---
describe('shouldExclude', () => {
test('excludes pseudo-page _atlas', () => {
expect(shouldExclude('_atlas')).toBe(true);
});
test('excludes pseudo-page _index', () => {
expect(shouldExclude('_index')).toBe(true);
});
test('excludes pseudo-page _stats', () => {
expect(shouldExclude('_stats')).toBe(true);
});
test('excludes pseudo-page _orphans', () => {
expect(shouldExclude('_orphans')).toBe(true);
});
test('excludes pseudo-page _scratch', () => {
expect(shouldExclude('_scratch')).toBe(true);
});
test('excludes pseudo-page claude', () => {
expect(shouldExclude('claude')).toBe(true);
});
test('excludes auto-generated _index suffix', () => {
expect(shouldExclude('companies/_index')).toBe(true);
expect(shouldExclude('people/_index')).toBe(true);
});
test('excludes auto-generated /log suffix', () => {
expect(shouldExclude('projects/acme/log')).toBe(true);
});
test('excludes raw source slugs', () => {
expect(shouldExclude('companies/acme/raw/crustdata')).toBe(true);
});
test('excludes deny-prefix: output/', () => {
expect(shouldExclude('output/2026-q1')).toBe(true);
});
test('excludes deny-prefix: dashboards/', () => {
expect(shouldExclude('dashboards/metrics')).toBe(true);
});
test('excludes deny-prefix: scripts/', () => {
expect(shouldExclude('scripts/ingest-runner')).toBe(true);
});
test('excludes deny-prefix: templates/', () => {
expect(shouldExclude('templates/meeting-note')).toBe(true);
});
test('excludes deny-prefix: openclaw/config/', () => {
expect(shouldExclude('openclaw/config/agent')).toBe(true);
});
test('excludes first-segment: scratch', () => {
expect(shouldExclude('scratch/idea-dump')).toBe(true);
});
test('excludes first-segment: thoughts', () => {
expect(shouldExclude('thoughts/2026-04-17')).toBe(true);
});
test('excludes first-segment: catalog', () => {
expect(shouldExclude('catalog/tools')).toBe(true);
});
test('excludes first-segment: entities', () => {
expect(shouldExclude('entities/product-hunt')).toBe(true);
});
test('does NOT exclude a normal content page', () => {
expect(shouldExclude('companies/acme')).toBe(false);
expect(shouldExclude('people/jane-doe')).toBe(false);
expect(shouldExclude('projects/gbrain')).toBe(false);
});
test('does NOT exclude a page ending with log-like text that is not /log', () => {
expect(shouldExclude('devlog')).toBe(false);
expect(shouldExclude('changelog')).toBe(false);
});
});
// --- deriveDomain ---
describe('deriveDomain', () => {
test('uses frontmatter domain when present', () => {
expect(deriveDomain('companies', 'companies/acme')).toBe('companies');
});
test('falls back to first slug segment', () => {
expect(deriveDomain(null, 'people/jane-doe')).toBe('people');
expect(deriveDomain(undefined, 'projects/gbrain')).toBe('projects');
});
test('returns root for single-segment slugs with no frontmatter', () => {
expect(deriveDomain(null, 'readme')).toBe('readme');
});
test('ignores empty-string frontmatter domain', () => {
expect(deriveDomain('', 'people/alice')).toBe('people');
});
test('ignores whitespace-only frontmatter domain', () => {
expect(deriveDomain(' ', 'people/alice')).toBe('people');
});
});
// --- formatOrphansText ---
describe('formatOrphansText', () => {
function makeResult(orphans: OrphanPage[], overrides?: Partial<OrphanResult>): OrphanResult {
return {
orphans,
total_orphans: orphans.length,
total_linkable: orphans.length + 50,
total_pages: orphans.length + 60,
excluded: 10,
...overrides,
};
}
test('shows summary line', () => {
const result = makeResult([]);
const out = formatOrphansText(result);
expect(out).toContain('0 orphans out of');
expect(out).toContain('total');
expect(out).toContain('excluded');
});
test('shows "No orphan pages found." when empty', () => {
const out = formatOrphansText(makeResult([]));
expect(out).toContain('No orphan pages found.');
});
test('groups orphans by domain', () => {
const orphans: OrphanPage[] = [
{ slug: 'companies/acme', title: 'Acme Corp', domain: 'companies' },
{ slug: 'people/alice', title: 'Alice', domain: 'people' },
{ slug: 'companies/beta', title: 'Beta Inc', domain: 'companies' },
];
const out = formatOrphansText(makeResult(orphans));
expect(out).toContain('[companies]');
expect(out).toContain('[people]');
// companies section should appear before people (alphabetical)
const companiesIdx = out.indexOf('[companies]');
const peopleIdx = out.indexOf('[people]');
expect(companiesIdx).toBeLessThan(peopleIdx);
});
test('sorts orphans alphabetically within each domain group', () => {
const orphans: OrphanPage[] = [
{ slug: 'companies/zeta', title: 'Zeta', domain: 'companies' },
{ slug: 'companies/alpha', title: 'Alpha', domain: 'companies' },
{ slug: 'companies/beta', title: 'Beta', domain: 'companies' },
];
const out = formatOrphansText(makeResult(orphans));
const alphaIdx = out.indexOf('companies/alpha');
const betaIdx = out.indexOf('companies/beta');
const zetaIdx = out.indexOf('companies/zeta');
expect(alphaIdx).toBeLessThan(betaIdx);
expect(betaIdx).toBeLessThan(zetaIdx);
});
test('includes slug and title in output', () => {
const orphans: OrphanPage[] = [
{ slug: 'companies/acme', title: 'Acme Corp', domain: 'companies' },
];
const out = formatOrphansText(makeResult(orphans));
expect(out).toContain('companies/acme');
expect(out).toContain('Acme Corp');
});
test('summary line shows correct numbers', () => {
const orphans: OrphanPage[] = [
{ slug: 'a/b', title: 'B', domain: 'a' },
{ slug: 'a/c', title: 'C', domain: 'a' },
];
const result: OrphanResult = {
orphans,
total_orphans: 2,
total_linkable: 100,
total_pages: 120,
excluded: 20,
};
const out = formatOrphansText(result);
expect(out).toContain('2 orphans out of 100 linkable pages (120 total; 20 excluded)');
});
});