diff --git a/src/cli.ts b/src/cli.ts index e1cb8dbe2..300f241c5 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -1396,6 +1396,14 @@ async function handleCliOnly(command: string, args: string[]) { return; } + // v0.42.x (#2390): `gbrain eval chronicle` is deterministic — brings its own + // in-memory PGLite, no DB/gateway. CI fixture gate runs anywhere. + if (command === 'eval' && args[0] === 'chronicle') { + const { runEvalChronicle } = await import('./commands/eval-chronicle.ts'); + setCliExitVerdict(await runEvalChronicle(args.slice(1))); + return; + } + // v0.41.13.0: `gbrain eval conversation-parser` is pure-function // (parses fixture JSONL, runs parseConversation, scores results). // No DB access; bypass connectEngine entirely so the CI fixture diff --git a/src/commands/eval-chronicle.ts b/src/commands/eval-chronicle.ts new file mode 100644 index 000000000..fe9bcfad7 --- /dev/null +++ b/src/commands/eval-chronicle.ts @@ -0,0 +1,44 @@ +// v0.42.x — Life Chronicle (#2390) `gbrain eval chronicle` (Phase A.9). +// Deterministic, brings its own in-memory PGLite (no DB, no gateway), so the +// CI fixture gate runs anywhere. Exit 0 only on a perfect score. +import { PGLiteEngine } from '../core/pglite-engine.ts'; +import { runChronicleEval } from '../eval/chronicle/harness.ts'; + +const HELP = `Usage: gbrain eval chronicle [--json] + +Deterministic Life Chronicle (#2390) feature eval. Builds a synthetic month +corpus with a known gold chronology + a planted ontology supersession + a +planted conflict, then scores the chronicle layer on: day reconstruction +(intra-day order), last-seen exact date, ontology supersession + --asof +time-travel, contradiction surfacing, and source isolation. + +Exit code 0 iff every task passes. +`; + +export async function runEvalChronicle(args: string[]): Promise { + if (args.includes('--help') || args.includes('-h')) { + process.stdout.write(HELP); + return 0; + } + const json = args.includes('--json'); + const engine = new PGLiteEngine(); + await engine.connect({ database_url: '' }); + await engine.initSchema(); + try { + const result = await runChronicleEval(engine); + if (json) { + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); + } else { + process.stderr.write( + `[eval chronicle] ${result.passed}/${result.total} tasks passed ` + + `(score ${(result.score * 100).toFixed(0)}%)\n`, + ); + for (const t of result.tasks) { + process.stderr.write(` ${t.passed ? 'PASS' : 'FAIL'} ${t.id} — ${t.detail}\n`); + } + } + return result.score === 1 ? 0 : 1; + } finally { + await engine.disconnect(); + } +} diff --git a/src/eval/chronicle/harness.ts b/src/eval/chronicle/harness.ts new file mode 100644 index 000000000..4b75f21ba --- /dev/null +++ b/src/eval/chronicle/harness.ts @@ -0,0 +1,89 @@ +// v0.42.x — Life Chronicle (#2390) feature eval (Phase A.9), the PRIMARY proof. +// +// Deterministic + CI-safe (no LLM): builds a small synthetic corpus with a KNOWN +// gold chronology + a planted ontology supersession + a planted conflict, then +// asserts the chronicle layer answers the temporal/longitudinal questions +// correctly. This is the "does the feature deliver value" bar from the North +// Star — chronology reconstruction, last-seen, supersession time-travel, +// contradiction surfacing, source isolation — measured against gold, not vibes. +// +// The OFF baseline (raw meeting pages) structurally CANNOT order intra-day +// events or time-travel ontology; the ON path (chronicle ops) does. We score +// the ON path against gold; a perfect score is the proof the ops are correct. +import type { BrainEngine } from '../../core/engine.ts'; +import { runChronicleExtract, type ChronicleJudge } from '../../core/chronicle/extract-events.ts'; + +export interface ChronicleEvalTask { id: string; passed: boolean; detail: string } +export interface ChronicleEvalResult { + tasks: ChronicleEvalTask[]; + passed: number; + total: number; + score: number; // passed / total, [0,1] +} + +const ALICE = 'people/alice-example'; +const BOB = 'people/bob-example'; + +/** Seed the synthetic corpus into a fresh engine (schema already initialized). */ +export async function seedChronicleEvalCorpus(engine: BrainEngine): Promise { + // Two same-day meetings → two events, gold intra-day order AM then PM. + await engine.putPage('meetings/2026-03-02-am', { + type: 'meeting', title: 'Morning standup', compiled_truth: 'x'.repeat(120), + frontmatter: { attendees: [ALICE] }, effective_date: new Date('2026-03-02T09:00:00Z'), + }); + await engine.putPage('meetings/2026-03-02-pm', { + type: 'meeting', title: 'Afternoon review', compiled_truth: 'y'.repeat(120), + frontmatter: { attendees: [ALICE] }, effective_date: new Date('2026-03-02T15:00:00Z'), + }); + const judge = (when: string, what: string): ChronicleJudge => async () => ({ + events: [{ when, who: [ALICE], what, kind: 'meeting' }], + }); + await runChronicleExtract(engine, { slug: 'meetings/2026-03-02-am', judge: judge('2026-03-02T09:00:00Z', 'Morning standup') }); + await runChronicleExtract(engine, { slug: 'meetings/2026-03-02-pm', judge: judge('2026-03-02T15:00:00Z', 'Afternoon review') }); + + // Ontology: alice was a founder, became an advisor (forward supersession). + await engine.mergeOntologyFact({ entitySlug: ALICE, dimension: 'role', value: 'founder', source: 'meetings/2024-01-10', validFrom: '2024-01-01' }); + await engine.mergeOntologyFact({ entitySlug: ALICE, dimension: 'role', value: 'advisor', source: 'meetings/2026-03-02-pm', validFrom: '2026-03-01' }); + + // Planted conflict: bob has two backdated-vs-forward open values from 2 sources. + await engine.mergeOntologyFact({ entitySlug: BOB, dimension: 'role', value: 'advisor', source: 'meetings/a', validFrom: '2026-05-01' }); + await engine.mergeOntologyFact({ entitySlug: BOB, dimension: 'role', value: 'founder', source: 'meetings/b', validFrom: '2026-01-01' }); +} + +export async function runChronicleEval(engine: BrainEngine): Promise { + await seedChronicleEvalCorpus(engine); + const tasks: ChronicleEvalTask[] = []; + const check = (id: string, ok: boolean, detail: string) => tasks.push({ id, passed: ok, detail }); + + // 1. Day reconstruction in correct intra-day order. + const day = await engine.getTimelineForDate('2026-03-02', { sourceId: 'default' }); + const order = day.map((r) => r.summary); + check('day_order', JSON.stringify(order) === JSON.stringify(['Morning standup', 'Afternoon review']), + `order=${JSON.stringify(order)}`); + + // 2. Last-seen exact date. + const ls = await engine.getLastSeen(ALICE, { sourceId: 'default' }); + check('last_seen', ls.last_date === '2026-03-02', `last_date=${ls.last_date}`); + + // 3. Supersession: current value is the new one. + const now = await engine.getOntology(ALICE, { sourceId: 'default' }); + const roleNow = now.find((v) => v.dimension === 'role')?.value; + check('supersession_now', roleNow === 'advisor', `role_now=${roleNow}`); + + // 4. Valid-time travel: as-of before the change returns the prior value. + const past = await engine.getOntology(ALICE, { sourceId: 'default', asof: '2025-01-01' }); + const rolePast = past.find((v) => v.dimension === 'role')?.value; + check('supersession_asof', rolePast === 'founder', `role_asof=${rolePast}`); + + // 5. Contradiction surfaced (genuine disagreement, not supersession). + const conflicts = await engine.findOntologyConflicts({ sourceId: 'default' }); + check('conflict_surfaced', conflicts.some((c) => c.entity_slug === BOB && c.dimension === 'role'), + `conflicts=${conflicts.length}`); + + // 6. Source isolation: querying another source leaks nothing. + const otherSource = await engine.getOntology(ALICE, { sourceId: 'nonexistent-source' }); + check('source_isolation', otherSource.length === 0, `leaked=${otherSource.length}`); + + const passed = tasks.filter((t) => t.passed).length; + return { tasks, passed, total: tasks.length, score: tasks.length ? passed / tasks.length : 0 }; +} diff --git a/test/eval-chronicle.test.ts b/test/eval-chronicle.test.ts new file mode 100644 index 000000000..cbb69222d --- /dev/null +++ b/test/eval-chronicle.test.ts @@ -0,0 +1,25 @@ +/** + * v0.42.x — Life Chronicle (#2390) feature eval harness (Phase A.9, PRIMARY proof). + * The chronicle layer must answer EVERY gold task on the synthetic corpus. + */ +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { PGLiteEngine } from '../src/core/pglite-engine.ts'; +import { runChronicleEval } from '../src/eval/chronicle/harness.ts'; + +let engine: PGLiteEngine; +beforeAll(async () => { + engine = new PGLiteEngine(); + await engine.connect({ database_url: '' }); + await engine.initSchema(); +}); +afterAll(async () => { await engine.disconnect(); }); + +describe('runChronicleEval', () => { + test('scores a perfect 6/6 on the synthetic corpus (the value proof)', async () => { + const result = await runChronicleEval(engine); + const failed = result.tasks.filter((t) => !t.passed).map((t) => `${t.id}: ${t.detail}`); + expect(failed).toEqual([]); // surfaces which gold task failed, if any + expect(result.score).toBe(1); + expect(result.total).toBe(6); + }); +});