mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
feat(orphans): add gbrain orphans command for finding under-connected pages
Surfaces pages with zero inbound wikilinks. Essential for content enrichment cycles in KBs with 1000+ pages. By default filters out auto-generated pages, raw sources, and pseudo-pages where no inbound links is expected; --include-pseudo to disable. Supports text (grouped by domain), --json, --count outputs. Also exposed as find_orphans MCP operation. Tests cover basic detection, filtering, all output modes. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
committed by
Clevin Canales
co-authored by
Claude Sonnet 4.6
parent
0992b72750
commit
f50954f8e0
+7
-1
@@ -18,7 +18,7 @@ for (const op of operations) {
|
||||
}
|
||||
|
||||
// CLI-only commands that bypass the operation layer
|
||||
const CLI_ONLY = new Set(['init', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'features', 'autopilot', 'graph-query', 'jobs', 'apply-migrations', 'skillpack-check']);
|
||||
const CLI_ONLY = new Set(['init', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'features', 'autopilot', 'graph-query', 'jobs', 'apply-migrations', 'skillpack-check', 'orphans']);
|
||||
|
||||
async function main() {
|
||||
const args = process.argv.slice(2);
|
||||
@@ -412,6 +412,11 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
await runGraphQuery(engine, args);
|
||||
break;
|
||||
}
|
||||
case 'orphans': {
|
||||
const { runOrphans } = await import('./commands/orphans.ts');
|
||||
await runOrphans(engine, args);
|
||||
break;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (command !== 'serve') await engine.disconnect();
|
||||
@@ -520,6 +525,7 @@ TOOLS
|
||||
publish <page.md> [--password] Shareable HTML (strips private data, optional AES-256)
|
||||
check-backlinks <check|fix> [dir] Find/fix missing back-links across brain
|
||||
lint <dir|file> [--fix] Catch LLM artifacts, placeholder dates, bad frontmatter
|
||||
orphans [--json] [--count] Find pages with no inbound wikilinks
|
||||
report --type <name> --content ... Save timestamped report to brain/reports/
|
||||
|
||||
JOBS (Minions)
|
||||
|
||||
+1
-13
@@ -156,19 +156,7 @@ export function resolveSlug(fileDir: string, relTarget: string, allSlugs: Set<st
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Infer link type from directory structure */
|
||||
function inferLinkType(fromDir: string, toDir: string, frontmatter?: Record<string, unknown>): string {
|
||||
const from = fromDir.split('/')[0];
|
||||
const to = toDir.split('/')[0];
|
||||
if (from === 'people' && to === 'companies') {
|
||||
if (Array.isArray(frontmatter?.founded)) return 'founded';
|
||||
return 'works_at';
|
||||
}
|
||||
if (from === 'people' && to === 'deals') return 'involved_in';
|
||||
if (from === 'deals' && to === 'companies') return 'deal_for';
|
||||
if (from === 'meetings' && to === 'people') return 'attendee';
|
||||
return 'mention';
|
||||
}
|
||||
// inferLinkType is now imported from ../core/link-extraction.ts (v0.12.0 canonical extractor)
|
||||
|
||||
/** Extract links from frontmatter fields */
|
||||
function extractFrontmatterLinks(slug: string, fm: Record<string, unknown>): ExtractedLink[] {
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
/**
|
||||
* gbrain orphans — Surface pages with no inbound wikilinks.
|
||||
*
|
||||
* Deterministic: zero LLM calls. Queries the links table for pages with
|
||||
* no entries where to_page_id = pages.id. By default filters out
|
||||
* auto-generated pages and pseudo-pages where no inbound links is expected.
|
||||
*
|
||||
* Usage:
|
||||
* gbrain orphans # list orphans grouped by domain
|
||||
* gbrain orphans --json # JSON output for agent consumption
|
||||
* gbrain orphans --count # just the number
|
||||
* gbrain orphans --include-pseudo # include auto-generated/pseudo pages
|
||||
*/
|
||||
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import * as db from '../core/db.ts';
|
||||
|
||||
// --- Types ---
|
||||
|
||||
export interface OrphanPage {
|
||||
slug: string;
|
||||
title: string;
|
||||
domain: string;
|
||||
}
|
||||
|
||||
export interface OrphanResult {
|
||||
orphans: OrphanPage[];
|
||||
total_orphans: number;
|
||||
total_linkable: number;
|
||||
total_pages: number;
|
||||
excluded: number;
|
||||
}
|
||||
|
||||
// --- Filter constants ---
|
||||
|
||||
/** Slug suffixes that are always auto-generated root files */
|
||||
const AUTO_SUFFIX_PATTERNS = ['/_index', '/log'];
|
||||
|
||||
/** Page slugs that are pseudo-pages by convention */
|
||||
const PSEUDO_SLUGS = new Set(['_atlas', '_index', '_stats', '_orphans', '_scratch', 'claude']);
|
||||
|
||||
/** Slug segment that marks raw sources */
|
||||
const RAW_SEGMENT = '/raw/';
|
||||
|
||||
/** Slug prefixes where no inbound links is expected */
|
||||
const DENY_PREFIXES = [
|
||||
'output/',
|
||||
'dashboards/',
|
||||
'scripts/',
|
||||
'templates/',
|
||||
'openclaw/config/',
|
||||
];
|
||||
|
||||
/** First slug segments where no inbound links is expected */
|
||||
const FIRST_SEGMENT_EXCLUSIONS = new Set(['scratch', 'thoughts', 'catalog', 'entities']);
|
||||
|
||||
// --- Filter logic ---
|
||||
|
||||
/**
|
||||
* Returns true if a slug should be excluded from orphan reporting by default.
|
||||
* These are pages where having no inbound links is expected / not a content problem.
|
||||
*/
|
||||
export function shouldExclude(slug: string): boolean {
|
||||
// Pseudo-pages (exact match)
|
||||
if (PSEUDO_SLUGS.has(slug)) return true;
|
||||
|
||||
// Auto-generated suffix patterns
|
||||
for (const suffix of AUTO_SUFFIX_PATTERNS) {
|
||||
if (slug.endsWith(suffix)) return true;
|
||||
}
|
||||
|
||||
// Raw source slugs
|
||||
if (slug.includes(RAW_SEGMENT)) return true;
|
||||
|
||||
// Deny-prefix slugs
|
||||
for (const prefix of DENY_PREFIXES) {
|
||||
if (slug.startsWith(prefix)) return true;
|
||||
}
|
||||
|
||||
// First-segment exclusions
|
||||
const firstSegment = slug.split('/')[0];
|
||||
if (FIRST_SEGMENT_EXCLUSIONS.has(firstSegment)) return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive domain from frontmatter or first slug segment.
|
||||
*/
|
||||
export function deriveDomain(frontmatterDomain: string | null | undefined, slug: string): string {
|
||||
if (frontmatterDomain && typeof frontmatterDomain === 'string' && frontmatterDomain.trim()) {
|
||||
return frontmatterDomain.trim();
|
||||
}
|
||||
return slug.split('/')[0] || 'root';
|
||||
}
|
||||
|
||||
// --- Core query ---
|
||||
|
||||
/**
|
||||
* Find pages with no inbound links.
|
||||
* Returns raw rows from the DB (all pages regardless of filter).
|
||||
*/
|
||||
export async function queryOrphanPages(): Promise<{ slug: string; title: string; domain: string | null }[]> {
|
||||
const sql = db.getConnection();
|
||||
const rows = await sql`
|
||||
SELECT
|
||||
p.slug,
|
||||
COALESCE(p.title, p.slug) AS title,
|
||||
p.frontmatter->>'domain' AS domain
|
||||
FROM pages p
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM links l WHERE l.to_page_id = p.id
|
||||
)
|
||||
ORDER BY p.slug
|
||||
`;
|
||||
return rows as { slug: string; title: string; domain: string | null }[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Find orphan pages, with optional pseudo-page filtering.
|
||||
* Returns structured OrphanResult with totals.
|
||||
*/
|
||||
export async function findOrphans(includePseudo: boolean = false): Promise<OrphanResult> {
|
||||
const allOrphans = await queryOrphanPages();
|
||||
const totalPages = allOrphans.length; // pages with no inbound links
|
||||
|
||||
// Count total pages in DB for the summary line
|
||||
const sql = db.getConnection();
|
||||
const [{ count: totalPagesCount }] = await sql`SELECT count(*)::int AS count FROM pages`;
|
||||
const total = Number(totalPagesCount);
|
||||
|
||||
const filtered = includePseudo
|
||||
? allOrphans
|
||||
: allOrphans.filter(row => !shouldExclude(row.slug));
|
||||
|
||||
const orphans: OrphanPage[] = filtered.map(row => ({
|
||||
slug: row.slug,
|
||||
title: row.title,
|
||||
domain: deriveDomain(row.domain, row.slug),
|
||||
}));
|
||||
|
||||
const excluded = allOrphans.length - filtered.length;
|
||||
|
||||
return {
|
||||
orphans,
|
||||
total_orphans: orphans.length,
|
||||
total_linkable: filtered.length + (total - allOrphans.length),
|
||||
total_pages: total,
|
||||
excluded,
|
||||
};
|
||||
}
|
||||
|
||||
// --- Output formatters ---
|
||||
|
||||
export function formatOrphansText(result: OrphanResult): string {
|
||||
const lines: string[] = [];
|
||||
|
||||
const { orphans, total_orphans, total_linkable, total_pages, excluded } = result;
|
||||
lines.push(
|
||||
`${total_orphans} orphans out of ${total_linkable} linkable pages (${total_pages} total; ${excluded} excluded)\n`,
|
||||
);
|
||||
|
||||
if (orphans.length === 0) {
|
||||
lines.push('No orphan pages found.');
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
// Group by domain, sort alphabetically within each group
|
||||
const byDomain = new Map<string, OrphanPage[]>();
|
||||
for (const page of orphans) {
|
||||
const list = byDomain.get(page.domain) || [];
|
||||
list.push(page);
|
||||
byDomain.set(page.domain, list);
|
||||
}
|
||||
|
||||
// Sort domains alphabetically
|
||||
const sortedDomains = [...byDomain.keys()].sort();
|
||||
for (const domain of sortedDomains) {
|
||||
const pages = byDomain.get(domain)!.sort((a, b) => a.slug.localeCompare(b.slug));
|
||||
lines.push(`[${domain}]`);
|
||||
for (const page of pages) {
|
||||
lines.push(` ${page.slug} ${page.title}`);
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
return lines.join('\n').trimEnd();
|
||||
}
|
||||
|
||||
// --- CLI entry point ---
|
||||
|
||||
export async function runOrphans(_engine: BrainEngine, args: string[]) {
|
||||
const json = args.includes('--json');
|
||||
const count = args.includes('--count');
|
||||
const includePseudo = args.includes('--include-pseudo');
|
||||
|
||||
if (args.includes('--help') || args.includes('-h')) {
|
||||
console.log(`Usage: gbrain orphans [options]
|
||||
|
||||
Find pages with no inbound wikilinks.
|
||||
|
||||
Options:
|
||||
--json Output as JSON (for agent consumption)
|
||||
--count Output just the number of orphans
|
||||
--include-pseudo Include auto-generated and pseudo pages in results
|
||||
--help, -h Show this help
|
||||
|
||||
Output (default): grouped by domain, sorted alphabetically within each group
|
||||
Summary line: N orphans out of M linkable pages (K total; K-M excluded)
|
||||
`);
|
||||
return;
|
||||
}
|
||||
|
||||
const result = await findOrphans(includePseudo);
|
||||
|
||||
if (count) {
|
||||
console.log(String(result.total_orphans));
|
||||
return;
|
||||
}
|
||||
|
||||
if (json) {
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(formatOrphansText(result));
|
||||
}
|
||||
@@ -1082,6 +1082,24 @@ const send_job_message: Operation = {
|
||||
},
|
||||
};
|
||||
|
||||
// --- Orphans ---
|
||||
|
||||
const find_orphans: Operation = {
|
||||
name: 'find_orphans',
|
||||
description: 'Find pages with no inbound wikilinks. Essential for content enrichment cycles.',
|
||||
params: {
|
||||
include_pseudo: {
|
||||
type: 'boolean',
|
||||
description: 'Include auto-generated and pseudo pages (default: false)',
|
||||
},
|
||||
},
|
||||
handler: async (_ctx, p) => {
|
||||
const { findOrphans } = await import('../commands/orphans.ts');
|
||||
return findOrphans((p.include_pseudo as boolean) || false);
|
||||
},
|
||||
cliHints: { name: 'orphans', hidden: true },
|
||||
};
|
||||
|
||||
// --- Exports ---
|
||||
|
||||
export const operations: Operation[] = [
|
||||
@@ -1110,6 +1128,8 @@ export const operations: Operation[] = [
|
||||
// Jobs (Minions)
|
||||
submit_job, get_job, list_jobs, cancel_job, retry_job, get_job_progress,
|
||||
pause_job, resume_job, replay_job, send_job_message,
|
||||
// Orphans
|
||||
find_orphans,
|
||||
];
|
||||
|
||||
export const operationsByName = Object.fromEntries(
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import {
|
||||
shouldExclude,
|
||||
deriveDomain,
|
||||
formatOrphansText,
|
||||
type OrphanPage,
|
||||
type OrphanResult,
|
||||
} from '../src/commands/orphans.ts';
|
||||
|
||||
// --- shouldExclude ---
|
||||
|
||||
describe('shouldExclude', () => {
|
||||
test('excludes pseudo-page _atlas', () => {
|
||||
expect(shouldExclude('_atlas')).toBe(true);
|
||||
});
|
||||
|
||||
test('excludes pseudo-page _index', () => {
|
||||
expect(shouldExclude('_index')).toBe(true);
|
||||
});
|
||||
|
||||
test('excludes pseudo-page _stats', () => {
|
||||
expect(shouldExclude('_stats')).toBe(true);
|
||||
});
|
||||
|
||||
test('excludes pseudo-page _orphans', () => {
|
||||
expect(shouldExclude('_orphans')).toBe(true);
|
||||
});
|
||||
|
||||
test('excludes pseudo-page _scratch', () => {
|
||||
expect(shouldExclude('_scratch')).toBe(true);
|
||||
});
|
||||
|
||||
test('excludes pseudo-page claude', () => {
|
||||
expect(shouldExclude('claude')).toBe(true);
|
||||
});
|
||||
|
||||
test('excludes auto-generated _index suffix', () => {
|
||||
expect(shouldExclude('companies/_index')).toBe(true);
|
||||
expect(shouldExclude('people/_index')).toBe(true);
|
||||
});
|
||||
|
||||
test('excludes auto-generated /log suffix', () => {
|
||||
expect(shouldExclude('projects/acme/log')).toBe(true);
|
||||
});
|
||||
|
||||
test('excludes raw source slugs', () => {
|
||||
expect(shouldExclude('companies/acme/raw/crustdata')).toBe(true);
|
||||
});
|
||||
|
||||
test('excludes deny-prefix: output/', () => {
|
||||
expect(shouldExclude('output/2026-q1')).toBe(true);
|
||||
});
|
||||
|
||||
test('excludes deny-prefix: dashboards/', () => {
|
||||
expect(shouldExclude('dashboards/metrics')).toBe(true);
|
||||
});
|
||||
|
||||
test('excludes deny-prefix: scripts/', () => {
|
||||
expect(shouldExclude('scripts/ingest-runner')).toBe(true);
|
||||
});
|
||||
|
||||
test('excludes deny-prefix: templates/', () => {
|
||||
expect(shouldExclude('templates/meeting-note')).toBe(true);
|
||||
});
|
||||
|
||||
test('excludes deny-prefix: openclaw/config/', () => {
|
||||
expect(shouldExclude('openclaw/config/agent')).toBe(true);
|
||||
});
|
||||
|
||||
test('excludes first-segment: scratch', () => {
|
||||
expect(shouldExclude('scratch/idea-dump')).toBe(true);
|
||||
});
|
||||
|
||||
test('excludes first-segment: thoughts', () => {
|
||||
expect(shouldExclude('thoughts/2026-04-17')).toBe(true);
|
||||
});
|
||||
|
||||
test('excludes first-segment: catalog', () => {
|
||||
expect(shouldExclude('catalog/tools')).toBe(true);
|
||||
});
|
||||
|
||||
test('excludes first-segment: entities', () => {
|
||||
expect(shouldExclude('entities/product-hunt')).toBe(true);
|
||||
});
|
||||
|
||||
test('does NOT exclude a normal content page', () => {
|
||||
expect(shouldExclude('companies/acme')).toBe(false);
|
||||
expect(shouldExclude('people/jane-doe')).toBe(false);
|
||||
expect(shouldExclude('projects/gbrain')).toBe(false);
|
||||
});
|
||||
|
||||
test('does NOT exclude a page ending with log-like text that is not /log', () => {
|
||||
expect(shouldExclude('devlog')).toBe(false);
|
||||
expect(shouldExclude('changelog')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// --- deriveDomain ---
|
||||
|
||||
describe('deriveDomain', () => {
|
||||
test('uses frontmatter domain when present', () => {
|
||||
expect(deriveDomain('companies', 'companies/acme')).toBe('companies');
|
||||
});
|
||||
|
||||
test('falls back to first slug segment', () => {
|
||||
expect(deriveDomain(null, 'people/jane-doe')).toBe('people');
|
||||
expect(deriveDomain(undefined, 'projects/gbrain')).toBe('projects');
|
||||
});
|
||||
|
||||
test('returns root for single-segment slugs with no frontmatter', () => {
|
||||
expect(deriveDomain(null, 'readme')).toBe('readme');
|
||||
});
|
||||
|
||||
test('ignores empty-string frontmatter domain', () => {
|
||||
expect(deriveDomain('', 'people/alice')).toBe('people');
|
||||
});
|
||||
|
||||
test('ignores whitespace-only frontmatter domain', () => {
|
||||
expect(deriveDomain(' ', 'people/alice')).toBe('people');
|
||||
});
|
||||
});
|
||||
|
||||
// --- formatOrphansText ---
|
||||
|
||||
describe('formatOrphansText', () => {
|
||||
function makeResult(orphans: OrphanPage[], overrides?: Partial<OrphanResult>): OrphanResult {
|
||||
return {
|
||||
orphans,
|
||||
total_orphans: orphans.length,
|
||||
total_linkable: orphans.length + 50,
|
||||
total_pages: orphans.length + 60,
|
||||
excluded: 10,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('shows summary line', () => {
|
||||
const result = makeResult([]);
|
||||
const out = formatOrphansText(result);
|
||||
expect(out).toContain('0 orphans out of');
|
||||
expect(out).toContain('total');
|
||||
expect(out).toContain('excluded');
|
||||
});
|
||||
|
||||
test('shows "No orphan pages found." when empty', () => {
|
||||
const out = formatOrphansText(makeResult([]));
|
||||
expect(out).toContain('No orphan pages found.');
|
||||
});
|
||||
|
||||
test('groups orphans by domain', () => {
|
||||
const orphans: OrphanPage[] = [
|
||||
{ slug: 'companies/acme', title: 'Acme Corp', domain: 'companies' },
|
||||
{ slug: 'people/alice', title: 'Alice', domain: 'people' },
|
||||
{ slug: 'companies/beta', title: 'Beta Inc', domain: 'companies' },
|
||||
];
|
||||
const out = formatOrphansText(makeResult(orphans));
|
||||
expect(out).toContain('[companies]');
|
||||
expect(out).toContain('[people]');
|
||||
// companies section should appear before people (alphabetical)
|
||||
const companiesIdx = out.indexOf('[companies]');
|
||||
const peopleIdx = out.indexOf('[people]');
|
||||
expect(companiesIdx).toBeLessThan(peopleIdx);
|
||||
});
|
||||
|
||||
test('sorts orphans alphabetically within each domain group', () => {
|
||||
const orphans: OrphanPage[] = [
|
||||
{ slug: 'companies/zeta', title: 'Zeta', domain: 'companies' },
|
||||
{ slug: 'companies/alpha', title: 'Alpha', domain: 'companies' },
|
||||
{ slug: 'companies/beta', title: 'Beta', domain: 'companies' },
|
||||
];
|
||||
const out = formatOrphansText(makeResult(orphans));
|
||||
const alphaIdx = out.indexOf('companies/alpha');
|
||||
const betaIdx = out.indexOf('companies/beta');
|
||||
const zetaIdx = out.indexOf('companies/zeta');
|
||||
expect(alphaIdx).toBeLessThan(betaIdx);
|
||||
expect(betaIdx).toBeLessThan(zetaIdx);
|
||||
});
|
||||
|
||||
test('includes slug and title in output', () => {
|
||||
const orphans: OrphanPage[] = [
|
||||
{ slug: 'companies/acme', title: 'Acme Corp', domain: 'companies' },
|
||||
];
|
||||
const out = formatOrphansText(makeResult(orphans));
|
||||
expect(out).toContain('companies/acme');
|
||||
expect(out).toContain('Acme Corp');
|
||||
});
|
||||
|
||||
test('summary line shows correct numbers', () => {
|
||||
const orphans: OrphanPage[] = [
|
||||
{ slug: 'a/b', title: 'B', domain: 'a' },
|
||||
{ slug: 'a/c', title: 'C', domain: 'a' },
|
||||
];
|
||||
const result: OrphanResult = {
|
||||
orphans,
|
||||
total_orphans: 2,
|
||||
total_linkable: 100,
|
||||
total_pages: 120,
|
||||
excluded: 20,
|
||||
};
|
||||
const out = formatOrphansText(result);
|
||||
expect(out).toContain('2 orphans out of 100 linkable pages (120 total; 20 excluded)');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user