/** * E4 — advisor ranking-precision eval. * * Measures the FEATURE (per the North Star): on synthetic, anonymized brains * with KNOWN seeded defects, does the advisor surface the right findings and * rank them correctly (critical > warn > info)? This tests the advisor's own * logic, not the remediation loop — each fixture injects defects through the * exact channels the collectors read (getStats/getHealth, minion_jobs, config), * then we score precision = (expected findings surfaced) / (expected total). * * Runs in the unit shard (under test/) so it's CI-gated; deterministic and * zero-API-cost (seeded fake engines), so it belongs with the unit suite rather * than a manual harness. Never ships downstream (test/ is not bundled). * * Synthetic only: no real brain content (anonymized fixtures), per the * calibration-corpus rule. */ import { describe, test, expect } from 'bun:test'; import { runAdvisor } from '../src/core/advisor/run.ts'; import type { AdvisorContext } from '../src/core/advisor/types.ts'; interface Fixture { name: string; engine: Partial; config?: Partial; remote?: boolean; /** Finding ids (or id prefixes) we expect to surface. */ expect: string[]; /** Finding ids that must NOT surface. */ forbid?: string[]; } const HEALTHY_STATS = { getStats: async () => ({ page_count: 500, chunk_count: 0, embedded_count: 0, link_count: 0, tag_count: 0, timeline_entry_count: 0, pages_by_type: {}, }), getHealth: async () => ({ page_count: 500, embed_coverage: 0.99, stale_pages: 0, orphan_pages: 0, missing_embeddings: 0, brain_score: 95, dead_links: 0, link_coverage: 1, timeline_coverage: 1, most_connected: [], embed_coverage_score: 35, link_density_score: 25, timeline_coverage_score: 15, no_orphans_score: 15, no_dead_links_score: 10, }), getConfig: async () => null, executeRaw: async () => [], }; const FIXTURES: Fixture[] = [ { name: 'healthy brain → no findings', engine: { ...HEALTHY_STATS }, expect: [], forbid: ['low_embed_coverage', 'orphan_pages', 'embeddings_disabled'], }, { name: 'degraded recall: low embed coverage + orphans', engine: { ...HEALTHY_STATS, getHealth: async () => ({ page_count: 500, embed_coverage: 0.3, stale_pages: 0, orphan_pages: 12, missing_embeddings: 350, brain_score: 40, dead_links: 2, link_coverage: 0.2, timeline_coverage: 0.2, most_connected: [], embed_coverage_score: 10, link_density_score: 5, timeline_coverage_score: 3, no_orphans_score: 2, no_dead_links_score: 8, }), }, expect: ['low_embed_coverage', 'orphan_pages', 'dead_links'], }, { name: 'stalled worker + stale source', engine: { ...HEALTHY_STATS, executeRaw: (async (sql: string) => { if (/minion_jobs/.test(sql)) return [{ name: 'embed-backfill', n: 3 }]; if (/sources/.test(sql)) return [{ id: 'wiki' }]; return []; }) as AdvisorContext['engine']['executeRaw'], }, expect: ['stalled_job:embed-backfill', 'stale_sync:wiki'], }, { name: 'setup smell: embeddings disabled', engine: { ...HEALTHY_STATS }, config: { embedding_disabled: true } as AdvisorContext['config'], expect: ['embeddings_disabled'], }, ]; function ctxFor(fx: Fixture): AdvisorContext { return { engine: fx.engine as AdvisorContext['engine'], config: (fx.config ?? {}) as AdvisorContext['config'], version: '0.43.0.0', workspace: null, skillsDir: null, now: new Date('2026-06-16T00:00:00Z'), remote: fx.remote ?? false, }; } describe('E4 advisor ranking-precision eval', () => { test('every seeded defect is surfaced and severities rank correctly', async () => { let expectedTotal = 0; let surfaced = 0; const SEV: Record = { critical: 0, warn: 1, info: 2 }; for (const fx of FIXTURES) { const report = await runAdvisor(ctxFor(fx)); const ids = report.findings.map((f) => f.id); for (const want of fx.expect) { expectedTotal++; if (ids.some((id) => id === want || id.startsWith(want))) surfaced++; else console.error(` [${fx.name}] MISSED expected finding: ${want}`); } for (const no of fx.forbid ?? []) { expect(ids.find((id) => id === no || id.startsWith(no))).toBeUndefined(); } // Ranking monotonic: severities never increase down the list. const sevs = report.findings.map((f) => SEV[f.severity]!); for (let i = 1; i < sevs.length; i++) expect(sevs[i]!).toBeGreaterThanOrEqual(sevs[i - 1]!); } const precision = expectedTotal === 0 ? 1 : surfaced / expectedTotal; console.error(`\nE4 advisor ranking precision: ${surfaced}/${expectedTotal} = ${(precision * 100).toFixed(0)}%`); expect(precision).toBe(1); }); test('remote advisor drops workspace-dependent findings (A1)', async () => { // A brain-pack uninstalled finding is workspace_dependent; over MCP it must // not appear even though the collector would fire locally. const report = await runAdvisor(ctxFor({ ...FIXTURES[0]!, remote: true })); expect(report.findings.every((f) => !f.workspace_dependent)).toBe(true); }); });