diff --git a/src/cli.ts b/src/cli.ts index 53622bb05..ecc81532a 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -54,7 +54,7 @@ export function bigintToStringReplacer(_key: string, value: unknown): unknown { } // CLI-only commands that bypass the operation layer -export const CLI_ONLY = new Set(['init', 'reinit-pglite', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'extract-conversation-facts', 'enrich', 'features', 'autopilot', 'graph-query', 'jobs', 'agent', 'apply-migrations', 'skillpack-check', 'skillpack', 'resolvers', 'integrity', 'repair-jsonb', 'orphans', 'sources', 'mounts', 'dream', 'check-resolvable', 'routing-eval', 'skillify', 'smoke-test', 'providers', 'storage', 'repos', 'code-def', 'code-refs', 'reindex', 'reindex-code', 'reindex-frontmatter', 'code-callers', 'code-callees', 'frontmatter', 'auth', 'friction', 'claw-test', 'book-mirror', 'takes', 'think', 'salience', 'anomalies', 'calibration', 'transcripts', 'models', 'remote', 'recall', 'forget', 'edges-backfill', 'cache', 'ze-switch', 'founder', 'brainstorm', 'lsd', 'schema', 'capture', 'onboard', 'conversation-parser', 'status', 'connect', 'skillopt', 'quarantine', 'self-upgrade', 'advisor', 'watch', 'reindex-search-vector']); +export const CLI_ONLY = new Set(['init', 'reinit-pglite', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'extract-conversation-facts', 'enrich', 'features', 'autopilot', 'graph-query', 'jobs', 'agent', 'apply-migrations', 'skillpack-check', 'skillpack', 'resolvers', 'integrity', 'repair-jsonb', 'orphans', 'maintain', 'sources', 'mounts', 'dream', 'check-resolvable', 'routing-eval', 'skillify', 'smoke-test', 'providers', 'storage', 'repos', 'code-def', 'code-refs', 'reindex', 'reindex-code', 'reindex-frontmatter', 'code-callers', 'code-callees', 'frontmatter', 'auth', 'friction', 'claw-test', 'book-mirror', 'takes', 'think', 'salience', 'anomalies', 'calibration', 'transcripts', 'models', 'remote', 'recall', 'forget', 'edges-backfill', 'cache', 'ze-switch', 'founder', 'brainstorm', 'lsd', 'schema', 'capture', 'onboard', 'conversation-parser', 'status', 'connect', 'skillopt', 'quarantine', 'self-upgrade', 'advisor', 'watch', 'reindex-search-vector']); // CLI-only commands whose handlers print their own --help text. These are // excluded from the generic short-circuit so detailed per-command and // per-subcommand usage stays reachable. @@ -78,6 +78,8 @@ const CLI_ONLY_SELF_HELP = new Set([ 'capture', // v0.42 self-upgrade ships its own usage (flags + the agent-skill story). 'self-upgrade', + // maintain (#3015) prints its own usage block (modes + not-auto-applied list). + 'maintain', // v0.43 (#2095): watch ships WATCH_HELP (flags + the stdin-turn protocol). 'watch', // v0.37 fix wave (Lane D.4 + CDX2-12): sync's --no-embed flag was @@ -1726,6 +1728,11 @@ async function handleCliOnly(command: string, args: string[]) { await runOrphans(engine, args); break; } + case 'maintain': { + const { runMaintain } = await import('./commands/maintain.ts'); + await runMaintain(engine, args); + break; + } // v0.32.7 CJK wave — post-upgrade markdown re-chunk sweep. // v0.36 Phase 3 wave — `gbrain reindex --multimodal` re-embeds content_chunks // into the unified Voyage multimodal-3 column. diff --git a/src/commands/extract.ts b/src/commands/extract.ts index 21eeaaef5..db232ec51 100644 --- a/src/commands/extract.ts +++ b/src/commands/extract.ts @@ -1651,7 +1651,7 @@ async function extractTimelineFromDB( * make re-extraction idempotent). EVERY processed page is stamped, including * zero-link pages — they WERE processed. */ -async function extractStaleFromDB( +export async function extractStaleFromDB( engine: BrainEngine, opts: { dryRun: boolean; diff --git a/src/commands/maintain.ts b/src/commands/maintain.ts new file mode 100644 index 000000000..2442b4844 --- /dev/null +++ b/src/commands/maintain.ts @@ -0,0 +1,224 @@ +/** + * gbrain maintain — conservative self-healing maintenance. + * + * This command automates the safe parts of the operator runbook: + * - stale link/timeline extraction + * - stale per-source dream cycles when doctor reports cycle_freshness + * + * It deliberately does NOT mutate source files, apply schema-pack upgrades, or + * invent semantic hub links. Those need review or a separate command with an + * auditable proposal surface. + */ + +import { existsSync } from 'fs'; +import type { BrainEngine } from '../core/engine.ts'; +import type { BrainHealth } from '../core/types.ts'; +import { buildChecks, computeDoctorReport, type DoctorReport, type Check } from './doctor.ts'; +import { extractStaleFromDB } from './extract.ts'; +import { runCycle, type CycleReport } from '../core/cycle.ts'; + +type ActionStatus = 'ok' | 'would_apply' | 'applied' | 'blocked' | 'skipped'; + +export interface MaintenanceAction { + name: string; + status: ActionStatus; + message: string; + details?: Record; +} + +export interface MaintainOptions { + json: boolean; + safe: boolean; + dryRun: boolean; + help: boolean; +} + +export interface MaintainReport { + mode: 'dry-run' | 'safe'; + before: { + health: BrainHealth; + doctor: DoctorReport; + }; + actions: MaintenanceAction[]; + after: { + health: BrainHealth; + doctor: DoctorReport; + }; +} + +export function parseMaintainArgs(args: string[]): MaintainOptions { + const safe = args.includes('--safe'); + return { + json: args.includes('--json'), + safe, + dryRun: args.includes('--dry-run') || !safe, + help: args.includes('--help') || args.includes('-h'), + }; +} + +export function extractCycleFreshnessSourceIds(checks: Check[]): string[] { + const ids = new Set(); + for (const check of checks) { + if (check.name !== 'cycle_freshness' || check.status === 'ok') continue; + const re = /Source '([^']+)' last cycled/g; + for (const match of check.message.matchAll(re)) { + const id = match[1]?.trim(); + if (id) ids.add(id); + } + } + return [...ids].sort(); +} + +async function buildDoctorReport(engine: BrainEngine): Promise { + const checks = await buildChecks(engine, ['--json', '--scope=brain']); + return computeDoctorReport(checks); +} + +async function runStaleExtraction( + engine: BrainEngine, + beforeHealth: BrainHealth, + dryRun: boolean, +): Promise { + if (beforeHealth.stale_pages <= 0) { + return { name: 'extract_stale', status: 'ok', message: 'No stale pages.' }; + } + + if (dryRun) { + return { + name: 'extract_stale', + status: 'would_apply', + message: `Would run DB-backed stale extraction for ${beforeHealth.stale_pages} page(s).`, + details: { stale_pages: beforeHealth.stale_pages }, + }; + } + + const result = await extractStaleFromDB(engine, { + dryRun: false, + jsonMode: false, + includeFrontmatter: false, + catchUp: false, + }); + + return { + name: 'extract_stale', + status: 'applied', + message: `Processed ${result.pagesProcessed} stale page(s); ${result.staleRemaining} remain.`, + details: { + links_created: result.linksCreated, + timeline_created: result.timelineCreated, + pages_processed: result.pagesProcessed, + stale_remaining: result.staleRemaining, + }, + }; +} + +async function runCycleFreshnessMaintenance( + engine: BrainEngine, + beforeDoctor: DoctorReport, + dryRun: boolean, +): Promise { + const sourceIds = extractCycleFreshnessSourceIds(beforeDoctor.checks); + if (sourceIds.length === 0) { + return [{ name: 'cycle_freshness', status: 'ok', message: 'All sources cycled recently.' }]; + } + + if (dryRun) { + return sourceIds.map((sourceId) => ({ + name: 'cycle_freshness', + status: 'would_apply', + message: `Would run source-scoped dream cycle for ${sourceId}.`, + details: { source_id: sourceId }, + })); + } + + const sources = await engine.listAllSources(); + const actions: MaintenanceAction[] = []; + + for (const sourceId of sourceIds) { + const source = sources.find((s) => s.id === sourceId); + const localPath = source?.local_path ?? null; + const brainDir = localPath && existsSync(localPath) ? localPath : null; + const report: CycleReport = await runCycle(engine, { + brainDir, + dryRun: false, + pull: false, + sourceId, + }); + actions.push({ + name: 'cycle_freshness', + status: report.status === 'failed' ? 'blocked' : 'applied', + message: `Ran source-scoped dream cycle for ${sourceId}: ${report.status}.`, + details: { + source_id: sourceId, + brain_dir: brainDir, + cycle_status: report.status, + phases: report.phases.map((p) => ({ phase: p.phase, status: p.status })), + }, + }); + } + + return actions; +} + +export async function runMaintain(engine: BrainEngine, args: string[]): Promise { + const opts = parseMaintainArgs(args); + if (opts.help) { + console.log(`Usage: gbrain maintain [--safe] [--dry-run] [--json] + +Conservative self-healing maintenance. + +Modes: + --dry-run Preview safe actions without writes. Default when --safe is absent. + --safe Apply safe actions: stale extraction and source cycle freshness. + --json Emit a structured before/action/after report. + +Not auto-applied: + source-file frontmatter fixes, schema-pack upgrades, atom-pack changes, + semantic hub-link guesses, and destructive cleanup. +`); + return; + } + + const beforeHealth = await engine.getHealth(); + const beforeDoctor = await buildDoctorReport(engine); + const actions: MaintenanceAction[] = []; + + actions.push(await runStaleExtraction(engine, beforeHealth, opts.dryRun)); + actions.push(...await runCycleFreshnessMaintenance(engine, beforeDoctor, opts.dryRun)); + + const afterHealth = await engine.getHealth(); + const afterDoctor = await buildDoctorReport(engine); + const report: MaintainReport = { + mode: opts.dryRun ? 'dry-run' : 'safe', + before: { health: beforeHealth, doctor: beforeDoctor }, + actions, + after: { health: afterHealth, doctor: afterDoctor }, + }; + + if (opts.json) { + console.log(JSON.stringify(report, null, 2)); + } else { + printMaintainReport(report); + } + return report; +} + +function printMaintainReport(report: MaintainReport): void { + console.log(`GBrain maintain (${report.mode})`); + console.log( + `Before: brain_score=${Math.round(report.before.health.brain_score)}/100 ` + + `stale=${report.before.health.stale_pages} islands=${report.before.health.orphan_pages} ` + + `doctor=${report.before.doctor.status}`, + ); + for (const action of report.actions) { + console.log(` ${action.status}: ${action.name} — ${action.message}`); + } + console.log( + `After: brain_score=${Math.round(report.after.health.brain_score)}/100 ` + + `stale=${report.after.health.stale_pages} islands=${report.after.health.orphan_pages} ` + + `doctor=${report.after.doctor.status}`, + ); + if (report.mode === 'dry-run') { + console.log('Run `gbrain maintain --safe` to apply safe actions.'); + } +} diff --git a/src/commands/orphans.ts b/src/commands/orphans.ts index a440c1017..645dd51e2 100644 --- a/src/commands/orphans.ts +++ b/src/commands/orphans.ts @@ -15,6 +15,11 @@ import type { BrainEngine } from '../core/engine.ts'; import { createProgress, startHeartbeat } from '../core/progress.ts'; import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts'; +import { + shouldExcludeFromOrphanReporting, + loadOrphanPolicyOverrides, + type OrphanPolicyOverrides, +} from '../core/orphan-policy.ts'; // --- Types --- @@ -32,65 +37,14 @@ export interface OrphanResult { excluded: number; } -// --- Filter constants --- - -/** Slug suffixes that are always auto-generated root files */ -const AUTO_SUFFIX_PATTERNS = ['/_index', '/log']; - -/** Page slugs that are pseudo-pages by convention */ -const PSEUDO_SLUGS = new Set(['_atlas', '_index', '_stats', '_orphans', '_scratch', 'claude']); - -/** Slug segment that marks raw sources */ -const RAW_SEGMENT = '/raw/'; - -/** Slug prefixes where no inbound links is expected */ -const DENY_PREFIXES = [ - 'output/', - 'dashboards/', - 'scripts/', - 'templates/', - 'openclaw/config/', -]; - -/** First slug segments where no inbound links is expected */ -const FIRST_SEGMENT_EXCLUSIONS = new Set([ - 'scratch', - 'thoughts', - 'catalog', - 'entities', - 'raw', - 'atoms', - 'skills', -]); - // --- Filter logic --- /** * Returns true if a slug should be excluded from orphan reporting by default. * These are pages where having no inbound links is expected / not a content problem. */ -export function shouldExclude(slug: string): boolean { - // Pseudo-pages (exact match) - if (PSEUDO_SLUGS.has(slug)) return true; - - // Auto-generated suffix patterns - for (const suffix of AUTO_SUFFIX_PATTERNS) { - if (slug.endsWith(suffix)) return true; - } - - // Raw source slugs - if (slug.includes(RAW_SEGMENT)) return true; - - // Deny-prefix slugs - for (const prefix of DENY_PREFIXES) { - if (slug.startsWith(prefix)) return true; - } - - // First-segment exclusions - const firstSegment = slug.split('/')[0]; - if (FIRST_SEGMENT_EXCLUSIONS.has(firstSegment)) return true; - - return false; +export function shouldExclude(slug: string, overrides?: OrphanPolicyOverrides): boolean { + return shouldExcludeFromOrphanReporting(slug, overrides); } /** @@ -156,6 +110,7 @@ export async function findOrphans( let allOrphans: { slug: string; title: string; domain: string | null }[]; let total: number; let excludedAll: number; + const overrides = includePseudo ? undefined : await loadOrphanPolicyOverrides(engine); try { allOrphans = await engine.findOrphanPages( sourceIds ? { sourceIds } : sourceId ? { sourceId } : undefined, @@ -184,7 +139,7 @@ export async function findOrphans( total = liveRows.length; excludedAll = includePseudo ? 0 - : liveRows.reduce((n, r) => n + (shouldExclude(r.slug) ? 1 : 0), 0); + : liveRows.reduce((n, r) => n + (shouldExclude(r.slug, overrides) ? 1 : 0), 0); } finally { stopHb(); progress.finish(); @@ -192,7 +147,7 @@ export async function findOrphans( const filtered = includePseudo ? allOrphans - : allOrphans.filter(row => !shouldExclude(row.slug)); + : allOrphans.filter(row => !shouldExclude(row.slug, overrides)); const orphans: OrphanPage[] = filtered.map(row => ({ slug: row.slug, diff --git a/src/core/orphan-policy.ts b/src/core/orphan-policy.ts new file mode 100644 index 000000000..321777ee8 --- /dev/null +++ b/src/core/orphan-policy.ts @@ -0,0 +1,116 @@ +/** + * Shared orphan-reporting exclusion policy. + * + * These are pages where "no inbound links" is expected and should not count + * against health. Keep this in core so the CLI orphan report and engine health + * dashboard cannot drift. + * + * Defaults are GBrain-wide conventions only. Brain-specific exclusions + * (private folder names, one-off fixture slugs) belong in the brain's own + * config, not here: + * + * gbrain config set orphans.exclude_prefixes "my-private-folder/,archive/" + * gbrain config set orphans.exclude_slugs "some-one-off-page" + */ + +const AUTO_SUFFIX_PATTERNS = ['/_index', '/log']; + +const PSEUDO_SLUGS = new Set(['_atlas', '_index', '_stats', '_orphans', '_scratch', 'claude']); + +const RAW_SEGMENT = '/raw/'; + +const DENY_PREFIXES = [ + 'output/', + 'dashboards/', + 'scripts/', + 'templates/', + '_templates/', + 'openclaw/config/', + 'extracts/', +]; + +const FIRST_SEGMENT_EXCLUSIONS = new Set([ + 'scratch', + 'thoughts', + 'catalog', + 'entities', + 'raw', + 'atoms', + 'skills', + 'dreaming', + 'daily', +]); + +const ROOT_DATE_SLUG = /^\d{4}-\d{2}-\d{2}(?:-.+)?$/; + +function isAgentWorkspaceConvention(slug: string): boolean { + if (!slug.startsWith('agents/')) return false; + if (slug.includes('/memory/dreaming/')) return true; + return /^agents\/[^/]+\/(?:agents|identity|soul|tools|user|heartbeat|dreams|dormant)$/.test(slug); +} + +/** Per-brain additions to the convention defaults (from config). */ +export interface OrphanPolicyOverrides { + excludePrefixes?: string[]; + excludeSlugs?: string[]; +} + +/** Config keys for per-brain orphan exclusions (comma-separated values). */ +export const ORPHAN_EXCLUDE_PREFIXES_KEY = 'orphans.exclude_prefixes'; +export const ORPHAN_EXCLUDE_SLUGS_KEY = 'orphans.exclude_slugs'; + +function parseList(value: string | null): string[] { + if (!value) return []; + return value.split(',').map(s => s.trim()).filter(Boolean); +} + +/** + * Load per-brain orphan exclusions from the brain config table. Callers with + * an engine in hand (getHealth, `gbrain orphans`) pass the result as the + * second argument to shouldExcludeFromOrphanReporting. + */ +export async function loadOrphanPolicyOverrides( + engine: { getConfig(key: string): Promise }, +): Promise { + const [prefixes, slugs] = await Promise.all([ + engine.getConfig(ORPHAN_EXCLUDE_PREFIXES_KEY), + engine.getConfig(ORPHAN_EXCLUDE_SLUGS_KEY), + ]); + return { excludePrefixes: parseList(prefixes), excludeSlugs: parseList(slugs) }; +} + +export function shouldExcludeFromOrphanReporting( + slug: string, + overrides?: OrphanPolicyOverrides, +): boolean { + if (PSEUDO_SLUGS.has(slug)) return true; + + for (const suffix of AUTO_SUFFIX_PATTERNS) { + if (slug.endsWith(suffix)) return true; + } + + if (slug.includes(RAW_SEGMENT)) return true; + if (slug.includes('/daily/')) return true; + + for (const prefix of DENY_PREFIXES) { + if (slug.startsWith(prefix)) return true; + } + + const firstSegment = slug.split('/')[0]; + if (FIRST_SEGMENT_EXCLUSIONS.has(firstSegment)) return true; + + if (ROOT_DATE_SLUG.test(slug)) return true; + + if (slug.startsWith('_brain-')) return true; + + if (isAgentWorkspaceConvention(slug)) return true; + + if (overrides) { + if (overrides.excludeSlugs?.includes(slug)) return true; + for (const prefix of overrides.excludePrefixes ?? []) { + if (slug.startsWith(prefix)) return true; + } + } + + return false; +} diff --git a/src/core/pglite-engine.ts b/src/core/pglite-engine.ts index 8239ec354..a5288a1b4 100644 --- a/src/core/pglite-engine.ts +++ b/src/core/pglite-engine.ts @@ -56,6 +56,8 @@ import { finalizeLastSeen } from './chronicle/last-seen.ts'; import { computeAnomaliesFromBuckets } from './cycle/anomaly.ts'; import { resolveBoostMap, resolveHardExcludes } from './search/source-boost.ts'; import { buildSourceFactorCase, buildHardExcludeClause, buildVisibilityClause, buildRecencyComponentSql, buildBestPerPagePoolCte } from './search/sql-ranking.ts'; +import { shouldExcludeFromOrphanReporting, loadOrphanPolicyOverrides } from './orphan-policy.ts'; +import { LINK_EXTRACTOR_VERSION_TS } from './link-extraction.ts'; import { normalizeEngineColumn, buildVectorCastFragment, @@ -5069,15 +5071,10 @@ export class PGLiteEngine implements BrainEngine { (SELECT count(*) FROM pages) as page_count, (SELECT count(*) FROM content_chunks WHERE embedded_at IS NOT NULL)::float / GREATEST((SELECT count(*) FROM content_chunks), 1)::float as embed_coverage, - (SELECT count(*) FROM pages p - WHERE p.updated_at < (SELECT MAX(te.created_at) FROM timeline_entries te WHERE te.page_id = p.id) - ) as stale_pages, - -- Bug 11 — orphan = islanded (no inbound AND no outbound). - -- See BrainHealth.orphan_pages docstring; docs updated to match this. - (SELECT count(*) FROM pages p - WHERE NOT EXISTS (SELECT 1 FROM links l WHERE l.to_page_id = p.id) - AND NOT EXISTS (SELECT 1 FROM links l WHERE l.from_page_id = p.id) - ) as orphan_pages, + 0 as stale_pages, + -- Bug 11 — orphan = islanded (no inbound AND no outbound). The raw + -- list is filtered in TS using the shared orphan-reporting policy. + 0 as orphan_pages, (SELECT count(*) FROM links l WHERE NOT EXISTS (SELECT 1 FROM pages p WHERE p.id = l.to_page_id) ) as dead_links, @@ -5102,10 +5099,20 @@ export class PGLiteEngine implements BrainEngine { LIMIT 5 `); + const { rows: islandedRows } = await this.db.query(` + SELECT p.slug + FROM pages p + WHERE NOT EXISTS (SELECT 1 FROM links l WHERE l.to_page_id = p.id) + AND NOT EXISTS (SELECT 1 FROM links l WHERE l.from_page_id = p.id) + `); + const r = h as Record; const pageCount = Number(r.page_count); const embedCoverage = Number(r.embed_coverage); - const orphanPages = Number(r.orphan_pages); + const stalePages = await this.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS }); + const orphanOverrides = await loadOrphanPolicyOverrides(this); + const orphanPages = (islandedRows as { slug: string }[]) + .filter(row => !shouldExcludeFromOrphanReporting(row.slug, orphanOverrides)).length; const deadLinks = Number(r.dead_links); const linkCount = Number(r.link_count); const pagesWithTimeline = Number(r.pages_with_timeline); @@ -5133,7 +5140,7 @@ export class PGLiteEngine implements BrainEngine { return { page_count: pageCount, embed_coverage: embedCoverage, - stale_pages: Number(r.stale_pages), + stale_pages: stalePages, orphan_pages: orphanPages, missing_embeddings: Number(r.missing_embeddings), brain_score: brainScore, diff --git a/src/core/postgres-engine.ts b/src/core/postgres-engine.ts index eafaf8bfa..623f20291 100644 --- a/src/core/postgres-engine.ts +++ b/src/core/postgres-engine.ts @@ -66,6 +66,8 @@ import { resolveBoostMap, resolveHardExcludes } from './search/source-boost.ts'; import { buildSourceFactorCase, buildHardExcludeClause, buildVisibilityClause, buildRecencyComponentSql, buildBestPerPagePoolCte } from './search/sql-ranking.ts'; import { DEFAULT_EMBEDDING_MODEL, DEFAULT_EMBEDDING_DIMENSIONS } from './ai/defaults.ts'; import { DELETE_BATCH_SIZE } from './engine-constants.ts'; +import { shouldExcludeFromOrphanReporting, loadOrphanPolicyOverrides } from './orphan-policy.ts'; +import { LINK_EXTRACTOR_VERSION_TS } from './link-extraction.ts'; function escapeSqlStringLiteral(value: string): string { return value.replace(/'/g, "''"); @@ -5152,11 +5154,9 @@ export class PostgresEngine implements BrainEngine { async getHealth(): Promise { const sql = this.sql; // Bug 11 doc-drift fix — orphan_pages means "islanded" (no inbound AND - // no outbound links), aligning both engines with the user-facing - // definition. The type comment previously said "no inbound" but the - // SQL required both — docs now match code so users can trust the - // number. A hub page that links out to many but has no back-references - // is working as intended, not an orphan. + // no outbound links). The raw islanded list is filtered through the same + // policy as `gbrain orphans` so convention pages do not count against + // dashboard health. const [h] = await sql` WITH entity_pages AS ( SELECT id, slug FROM pages WHERE type IN ('person', 'company') @@ -5165,13 +5165,8 @@ export class PostgresEngine implements BrainEngine { (SELECT count(*) FROM pages) as page_count, (SELECT count(*) FROM content_chunks WHERE embedded_at IS NOT NULL)::float / GREATEST((SELECT count(*) FROM content_chunks), 1)::float as embed_coverage, - (SELECT count(*) FROM pages p - WHERE p.updated_at < (SELECT MAX(te.created_at) FROM timeline_entries te WHERE te.page_id = p.id) - ) as stale_pages, - (SELECT count(*) FROM pages p - WHERE NOT EXISTS (SELECT 1 FROM links l WHERE l.to_page_id = p.id) - AND NOT EXISTS (SELECT 1 FROM links l WHERE l.from_page_id = p.id) - ) as orphan_pages, + 0 as stale_pages, + 0 as orphan_pages, (SELECT count(*) FROM links l WHERE NOT EXISTS (SELECT 1 FROM pages p WHERE p.id = l.to_page_id) ) as dead_links, @@ -5195,9 +5190,18 @@ export class PostgresEngine implements BrainEngine { LIMIT 5 `; + const islandedRows = await sql<{ slug: string }[]>` + SELECT p.slug + FROM pages p + WHERE NOT EXISTS (SELECT 1 FROM links l WHERE l.to_page_id = p.id) + AND NOT EXISTS (SELECT 1 FROM links l WHERE l.from_page_id = p.id) + `; + const pageCount = Number(h.page_count); const embedCoverage = Number(h.embed_coverage); - const orphanPages = Number(h.orphan_pages); + const stalePages = await this.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS }); + const orphanOverrides = await loadOrphanPolicyOverrides(this); + const orphanPages = islandedRows.filter(row => !shouldExcludeFromOrphanReporting(row.slug, orphanOverrides)).length; const deadLinks = Number(h.dead_links); const linkCount = Number(h.link_count); const pagesWithTimeline = Number(h.pages_with_timeline); @@ -5225,7 +5229,7 @@ export class PostgresEngine implements BrainEngine { return { page_count: pageCount, embed_coverage: embedCoverage, - stale_pages: Number(h.stale_pages), + stale_pages: stalePages, orphan_pages: orphanPages, missing_embeddings: Number(h.missing_embeddings), brain_score: brainScore, diff --git a/test/maintain.test.ts b/test/maintain.test.ts new file mode 100644 index 000000000..0db805a87 --- /dev/null +++ b/test/maintain.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, test } from 'bun:test'; +import { + extractCycleFreshnessSourceIds, + parseMaintainArgs, +} from '../src/commands/maintain.ts'; +import type { Check } from '../src/commands/doctor.ts'; + +describe('maintain args', () => { + test('defaults to dry-run unless --safe is explicit', () => { + expect(parseMaintainArgs([])).toMatchObject({ + safe: false, + dryRun: true, + json: false, + }); + }); + + test('--safe enables mutating safe mode', () => { + expect(parseMaintainArgs(['--safe', '--json'])).toMatchObject({ + safe: true, + dryRun: false, + json: true, + }); + }); + + test('--dry-run wins over --safe', () => { + expect(parseMaintainArgs(['--safe', '--dry-run'])).toMatchObject({ + safe: true, + dryRun: true, + }); + }); +}); + +describe('cycle freshness source extraction', () => { + test('extracts stale source ids from doctor messages', () => { + const checks: Check[] = [ + { + name: 'cycle_freshness', + status: 'fail', + message: "Source 'brain-sync-remote-teffur' last cycled 40h ago. Run `gbrain dream --source `.", + }, + { + name: 'cycle_freshness', + status: 'fail', + message: "Source 'wiki' last cycled 25h ago. Source 'wiki' last cycled 25h ago.", + }, + ]; + + expect(extractCycleFreshnessSourceIds(checks)).toEqual([ + 'brain-sync-remote-teffur', + 'wiki', + ]); + }); + + test('ignores ok and unrelated checks', () => { + const checks: Check[] = [ + { name: 'cycle_freshness', status: 'ok', message: "Source 'fresh' last cycled recently." }, + { name: 'frontmatter_integrity', status: 'warn', message: "Source 'wiki' has frontmatter issues." }, + ]; + + expect(extractCycleFreshnessSourceIds(checks)).toEqual([]); + }); +}); diff --git a/test/orphans-pure-fn.test.ts b/test/orphans-pure-fn.test.ts index ada6a7d09..79df84b79 100644 --- a/test/orphans-pure-fn.test.ts +++ b/test/orphans-pure-fn.test.ts @@ -186,11 +186,67 @@ describe('shouldExclude — orphan filter regression (preserve curation)', () => expect(shouldExclude('entities/anonymous')).toBe(true); expect(shouldExclude('atoms/fact-123')).toBe(true); expect(shouldExclude('skills/gbrain-operations')).toBe(true); + expect(shouldExclude('dreaming/light/2026-07-20')).toBe(true); + expect(shouldExclude('daily/2026-07-20')).toBe(true); + expect(shouldExclude('agent-openclaw/daily/2026-07-20')).toBe(true); + }); + + test('workspace convention slugs are excluded', () => { + expect(shouldExclude('_brain-conventions')).toBe(true); + expect(shouldExclude('_templates/decision')).toBe(true); + expect(shouldExclude('extracts/2026-06-30/takes.proposed/round-single')).toBe(true); + expect(shouldExclude('2026-07-20')).toBe(true); + expect(shouldExclude('2026-07-20-qa-sweep')).toBe(true); + expect(shouldExclude('agents/arya/identity')).toBe(true); + expect(shouldExclude('agents/arya/memory/dreaming/deep/2026-07-20')).toBe(true); }); test('regular slugs are NOT excluded', () => { expect(shouldExclude('people/alice')).toBe(false); expect(shouldExclude('companies/acme')).toBe(false); expect(shouldExclude('writing/post-1')).toBe(false); + expect(shouldExclude('agents/arya/qa-reports/launch-review')).toBe(false); + }); +}); + +describe('getHealth orphan_pages uses shared exclusion policy', () => { + test('excluded convention islands do not count against health', async () => { + await engine.putPage('_templates/decision', { + type: 'template', title: 'Decision', compiled_truth: 'template', timeline: '', frontmatter: {}, + }); + await engine.putPage('skills/arya/source-check', { + type: 'concept', title: 'Skill', compiled_truth: 'skill', timeline: '', frontmatter: {}, + }); + await engine.putPage('agents/arya/identity', { + type: 'note', title: 'Identity', compiled_truth: 'identity', timeline: '', frontmatter: {}, + }); + await engine.putPage('people/alice', { + type: 'person', title: 'Alice', compiled_truth: 'real island', timeline: '', frontmatter: {}, + }); + + const health = await engine.getHealth(); + + expect(health.orphan_pages).toBe(1); + }); + + test('per-brain config overrides (orphans.exclude_*) also apply to health', async () => { + await engine.putPage('my-private-folder/secret-ref', { + type: 'note', title: 'Ref', compiled_truth: 'ref', timeline: '', frontmatter: {}, + }); + await engine.putPage('one-off-fixture-page', { + type: 'note', title: 'Fixture', compiled_truth: 'fixture', timeline: '', frontmatter: {}, + }); + await engine.putPage('people/alice', { + type: 'person', title: 'Alice', compiled_truth: 'real island', timeline: '', frontmatter: {}, + }); + + expect((await engine.getHealth()).orphan_pages).toBe(3); + + await engine.setConfig('orphans.exclude_prefixes', 'my-private-folder/'); + await engine.setConfig('orphans.exclude_slugs', 'one-off-fixture-page'); + expect((await engine.getHealth()).orphan_pages).toBe(1); + + await engine.unsetConfig('orphans.exclude_prefixes'); + await engine.unsetConfig('orphans.exclude_slugs'); }); }); diff --git a/test/orphans.test.ts b/test/orphans.test.ts index 7d56bce01..ecbfb74f0 100644 --- a/test/orphans.test.ts +++ b/test/orphans.test.ts @@ -66,6 +66,10 @@ describe('shouldExclude', () => { expect(shouldExclude('templates/meeting-note')).toBe(true); }); + test('excludes deny-prefix: _templates/', () => { + expect(shouldExclude('_templates/meeting-note')).toBe(true); + }); + test('excludes deny-prefix: openclaw/config/', () => { expect(shouldExclude('openclaw/config/agent')).toBe(true); }); @@ -86,10 +90,44 @@ describe('shouldExclude', () => { expect(shouldExclude('entities/product-hunt')).toBe(true); }); + test('excludes first-segment: skills, dreaming, and daily', () => { + expect(shouldExclude('skills/arya/source-check')).toBe(true); + expect(shouldExclude('dreaming/light/2026-07-20')).toBe(true); + expect(shouldExclude('daily/2026-07-20')).toBe(true); + expect(shouldExclude('agent-openclaw/daily/2026-07-20')).toBe(true); + }); + + test('excludes root date logs and agent workspace conventions', () => { + expect(shouldExclude('_brain-conventions')).toBe(true); + expect(shouldExclude('2026-07-20')).toBe(true); + expect(shouldExclude('2026-07-20-qa-sweep')).toBe(true); + expect(shouldExclude('agents/arya/identity')).toBe(true); + expect(shouldExclude('agents/arya/memory/dreaming/deep/2026-07-20')).toBe(true); + }); + + test('excludes generated extracts', () => { + expect(shouldExclude('extracts/2026-06-30/takes.proposed/round-single')).toBe(true); + }); + + test('brain-specific exclusions come from config overrides, not global defaults', () => { + // No baked-in defaults for these: + expect(shouldExclude('my-private-folder/some-secret-ref.md')).toBe(false); + expect(shouldExclude('one-off-fixture-page')).toBe(false); + // The per-brain config plane (orphans.exclude_prefixes / exclude_slugs): + const overrides = { + excludePrefixes: ['my-private-folder/'], + excludeSlugs: ['one-off-fixture-page'], + }; + expect(shouldExclude('my-private-folder/some-secret-ref.md', overrides)).toBe(true); + expect(shouldExclude('one-off-fixture-page', overrides)).toBe(true); + expect(shouldExclude('people/jane-doe', overrides)).toBe(false); + }); + test('does NOT exclude a normal content page', () => { expect(shouldExclude('companies/acme')).toBe(false); expect(shouldExclude('people/jane-doe')).toBe(false); expect(shouldExclude('projects/gbrain')).toBe(false); + expect(shouldExclude('agents/arya/qa-reports/launch-review')).toBe(false); }); test('does NOT exclude a page ending with log-like text that is not /log', () => {