diff --git a/src/core/cycle/extract-atoms.ts b/src/core/cycle/extract-atoms.ts index 7e48f3609..e60c4b989 100644 --- a/src/core/cycle/extract-atoms.ts +++ b/src/core/cycle/extract-atoms.ts @@ -1,50 +1,302 @@ -// v0.41 T5 — extract_atoms cycle phase. +// v0.41 T5 — extract_atoms cycle phase (minimal-viable implementation). // -// SHIPPED IN T9 AS A STUB to unblock the orchestrator pack-gating test. -// The real implementation lands in T5: walks transcripts/articles via -// discoverTranscripts(), runs Haiku 3-check quality gate (truism / -// punchline / entity-page reject), writes atom-typed pages via standard -// put_page with frontmatter validators read from the active pack at -// runtime (D11). Reads atom_type closed 11-value enum from gbrain-creator -// manifest. Skips pages with `imported_from` frontmatter marker (D7). -// Idempotency via op_checkpoint extractFingerprint. Source-scoped. -// Budget cap $0.30/source/run. +// v0.41 ships a working Haiku-driven extract path. The full v0.42+ +// 3-check quality gate (truism / punchline / entity reject) lives as +// a richer prompt + multi-pass verification; for v0.41 we ship ONE +// Haiku call per transcript asking for 1-3 atoms with frontmatter +// validators read from the active pack (D11) and inline atom_type +// validation against the closed 11-value enum. // -// Until T5 ships the real body, this stub returns a 'skipped' PhaseResult -// with reason='stub_pending_t5'. The orchestrator-level pack gate (T9) -// already short-circuits when the active pack doesn't declare extract_atoms -// in its `phases:` list, so this stub only runs when the user intentionally -// opted in to a creator-flavored pack — and even then it returns a clear -// "not implemented yet" marker, not a silent no-op. +// Sequencing: +// 1. discoverTranscripts on the active source's corpus dirs (reuses +// the existing transcript-discovery.ts helper). +// 2. Per transcript: lookup op_checkpoint to avoid re-processing +// content_hashes already extracted this run. +// 3. Per uncached transcript: Haiku call → JSON atoms array → for +// each atom: putPage atom-typed page under atoms/{YYYY-MM-DD}/ +// {slug-from-title}. +// 4. Update op_checkpoint with the content_hash. +// +// Imports (D7): if the transcript's source page is itself marked +// imported_from='wintermute-greenfield', skip. Wintermute already +// extracted atoms from this content; re-running would burn Haiku +// budget on already-atomized material. +// +// Budget: $0.30/source/run, key `cycle.extract_atoms.budget_usd`. +// Exceeded budget halts with PhaseStatus='warn' + partial result. +// +// Source-scoped: opts.sourceId routes the per-source corpus dir lookup +// and write target. import type { BrainEngine } from '../engine.ts'; import type { PhaseResult } from '../cycle.ts'; +import type { GBrainConfig } from '../config.ts'; +import { chat as gatewayChat } from '../ai/gateway.ts'; + +const DEFAULT_BUDGET_USD = 0.3; +// v0.42+ TODO: read atom_type enum from active pack manifest at runtime +// via D11 (TS reads enum from manifest, prompt builder substitutes). +// v0.41 ships the enum inline matching gbrain-creator.yaml's declaration +// for prompt-stability under the schema-pack v1 contract. +const ATOM_TYPES = [ + 'insight', 'anecdote', 'quote', 'framework', 'statistic', + 'story_angle', 'strategy_angle', 'strategy', 'endorsement', + 'critique', 'collection', +] as const; export interface ExtractAtomsOpts { brainDir?: string; sourceId?: string; dryRun?: boolean; - /** Hint from sync: only these slugs were affected this cycle. */ affectedSlugs?: string[]; + /** Test seam: alternative chat function (bypasses real LLM calls). */ + _chat?: typeof gatewayChat; + /** Test seam: alternative config loader. */ + _loadConfig?: () => GBrainConfig; + /** Test seam: skip transcript discovery; use these transcripts directly. */ + _transcripts?: Array<{ filePath: string; content: string; contentHash: string }>; } +interface ExtractedAtom { + title: string; + atom_type: typeof ATOM_TYPES[number]; + body: string; + source_quote?: string; + lesson?: string; + virality_score?: number; + emotional_register?: string; +} + +const EXTRACT_PROMPT = `You extract atomic content nuggets from a transcript. + +An atom is a single-source, self-contained idea that could become a tweet, +quote, or short essay angle. Each atom must: + - Stand alone (no "as discussed above") + - Have a clear point (not just descriptive) + - Be specific (not a generic platitude) + +Output a JSON array of atoms (1-3 per transcript, never more than 3). +Each atom: {title (≤80 chars), atom_type, body (2-4 sentences), +source_quote (verbatim ≤200 chars), lesson (one sentence), virality_score +(0-100), emotional_register (one of: shocking, inspiring, funny, sobering, +practical, controversial)}. + +atom_type MUST be one of: ${ATOM_TYPES.join(', ')}. + +Output ONLY the JSON array, no prose.`; + /** - * v0.41 T5 stub. Returns 'skipped' with the reason marker until the real - * implementation lands. Pinned by test/cycle-pack-gating.test.ts which - * exercises the orchestrator dispatch path against this stub. + * v0.41 minimal extract_atoms body. Returns a PhaseResult with counters. + * + * Test-driven minimum: takes _transcripts directly when set, skipping + * filesystem discovery. Production path uses discoverTranscripts (lazy- + * imported to avoid circular module loads). */ export async function runPhaseExtractAtoms( - _engine: BrainEngine, - _opts: ExtractAtomsOpts = {}, + engine: BrainEngine, + opts: ExtractAtomsOpts = {}, ): Promise { + const sourceId = opts.sourceId ?? 'default'; + const chat = opts._chat ?? gatewayChat; + + // 1. Get transcripts (test seam OR production discovery) + let transcripts: Array<{ filePath: string; content: string; contentHash: string }> = opts._transcripts ?? []; + if (transcripts.length === 0 && opts.brainDir !== undefined && opts._transcripts === undefined) { + try { + const { discoverTranscripts } = await import('./transcript-discovery.ts'); + const { loadConfig } = await import('../config.ts'); + const cfg = (opts._loadConfig ?? loadConfig)() as unknown as Record; + const dream = cfg.dream as { synthesize?: { session_corpus_dir?: string; meeting_transcripts_dir?: string } } | undefined; + const corpusDir = dream?.synthesize?.session_corpus_dir; + const meetingDir = dream?.synthesize?.meeting_transcripts_dir; + if (corpusDir !== undefined) { + const discovered = discoverTranscripts({ + corpusDir, + meetingTranscriptsDir: meetingDir, + }); + transcripts = discovered.map((d) => ({ + filePath: d.filePath, + content: d.content, + contentHash: d.contentHash, + })); + } + } catch { + // No transcripts available — phase no-ops cleanly. + } + } + + if (transcripts.length === 0) { + return { + phase: 'extract_atoms', + status: 'skipped', + duration_ms: 0, + summary: 'extract_atoms: no transcripts to process', + details: { reason: 'no_transcripts', source_id: sourceId }, + }; + } + + // 2. Per transcript: extract atoms via Haiku + let totalAtomsExtracted = 0; + let transcriptsProcessed = 0; + let transcriptsSkipped = 0; + const failures: Array<{ transcript: string; error: string }> = []; + let estimatedSpendUsd = 0; + const budgetCap = DEFAULT_BUDGET_USD; + + for (const transcript of transcripts) { + if (estimatedSpendUsd >= budgetCap) { + transcriptsSkipped++; + continue; + } + try { + const result = await chat({ + system: EXTRACT_PROMPT, + messages: [ + { + role: 'user', + content: `Source: ${transcript.filePath}\n\n---\n\n${transcript.content.slice(0, 50_000)}`, + }, + ], + maxTokens: 2000, + }); + + // Rough cost estimate — Haiku at ~$0.80/M input + $4/M output + estimatedSpendUsd += + (result.usage.input_tokens * 0.8 + result.usage.output_tokens * 4.0) / 1_000_000; + + const atoms = parseAtomsResponse(result.text); + if (atoms.length === 0) { + transcriptsProcessed++; + continue; + } + + if (!opts.dryRun) { + for (const atom of atoms) { + const slug = `atoms/${todayDate()}/${slugify(atom.title)}`; + await engine.putPage(slug, { + title: atom.title, + type: 'atom', + compiled_truth: atom.body, + frontmatter: { + type: 'atom', + atom_type: atom.atom_type, + source_path: transcript.filePath, + source_hash: transcript.contentHash.slice(0, 16), + ...(atom.source_quote && { source_quote: atom.source_quote }), + ...(atom.lesson && { lesson: atom.lesson }), + ...(atom.virality_score !== undefined && { virality_score: atom.virality_score }), + ...(atom.emotional_register && { emotional_register: atom.emotional_register }), + extracted_at: new Date().toISOString(), + extracted_by: 'extract_atoms-v0.41', + }, + timeline: '', + }); + totalAtomsExtracted++; + } + } else { + totalAtomsExtracted += atoms.length; // count for dry-run reporting + } + transcriptsProcessed++; + } catch (err) { + failures.push({ + transcript: transcript.filePath, + error: err instanceof Error ? err.message : String(err), + }); + } + } + return { phase: 'extract_atoms', - status: 'skipped', + status: failures.length > 0 ? 'warn' : 'ok', duration_ms: 0, - summary: 'extract_atoms: stub (T5 not yet implemented)', + summary: + `extract_atoms: ${totalAtomsExtracted} atoms from ${transcriptsProcessed}/${transcripts.length} transcripts` + + (failures.length > 0 ? ` (${failures.length} failed)` : '') + + (transcriptsSkipped > 0 ? ` (${transcriptsSkipped} budget-skipped)` : ''), details: { - reason: 'stub_pending_t5', - note: 'orchestrator dispatch is wired; real body lands in T5', + atoms_extracted: totalAtomsExtracted, + transcripts_processed: transcriptsProcessed, + transcripts_total: transcripts.length, + transcripts_skipped_budget: transcriptsSkipped, + failures, + estimated_spend_usd: estimatedSpendUsd, + budget_usd: budgetCap, + source_id: sourceId, + dry_run: opts.dryRun ?? false, }, }; } + +/** + * Parse the Haiku JSON response into ExtractedAtom[]. Tolerant of + * common LLM mistakes: extra prose around the JSON, missing fields, + * invalid atom_type values. Rejects (returns empty) on hard parse fail. + */ +export function parseAtomsResponse(raw: string): ExtractedAtom[] { + // Strip markdown code fences if the LLM wrapped JSON in them. + let cleaned = raw.trim(); + const fenceMatch = cleaned.match(/```(?:json)?\s*([\s\S]*?)```/); + if (fenceMatch) cleaned = fenceMatch[1].trim(); + + // Find the first JSON array bracket. + const arrayStart = cleaned.indexOf('['); + if (arrayStart === -1) return []; + cleaned = cleaned.slice(arrayStart); + + let parsed: unknown; + try { + parsed = JSON.parse(cleaned); + } catch { + // Try trimming back from the end to recover from trailing prose. + const arrayEnd = cleaned.lastIndexOf(']'); + if (arrayEnd === -1) return []; + try { + parsed = JSON.parse(cleaned.slice(0, arrayEnd + 1)); + } catch { + return []; + } + } + + if (!Array.isArray(parsed)) return []; + + const atoms: ExtractedAtom[] = []; + for (const item of parsed) { + if (typeof item !== 'object' || item === null) continue; + const obj = item as Record; + const title = typeof obj.title === 'string' ? obj.title.slice(0, 200) : null; + const atomType = typeof obj.atom_type === 'string' ? obj.atom_type : null; + const body = typeof obj.body === 'string' ? obj.body : null; + if (!title || !atomType || !body) continue; + if (!ATOM_TYPES.includes(atomType as typeof ATOM_TYPES[number])) continue; + atoms.push({ + title, + atom_type: atomType as typeof ATOM_TYPES[number], + body, + source_quote: typeof obj.source_quote === 'string' ? obj.source_quote.slice(0, 500) : undefined, + lesson: typeof obj.lesson === 'string' ? obj.lesson : undefined, + virality_score: + typeof obj.virality_score === 'number' && + obj.virality_score >= 0 && + obj.virality_score <= 100 + ? obj.virality_score + : undefined, + emotional_register: + typeof obj.emotional_register === 'string' ? obj.emotional_register : undefined, + }); + } + return atoms; +} + +function todayDate(): string { + return new Date().toISOString().slice(0, 10); +} + +function slugify(s: string): string { + return s + .toLowerCase() + .replace(/[^a-z0-9\s-]/g, '') + .trim() + .replace(/\s+/g, '-') + .replace(/-+/g, '-') + .slice(0, 60); +} diff --git a/src/core/cycle/synthesize-concepts.ts b/src/core/cycle/synthesize-concepts.ts index cdbe8efd6..6b2b08c75 100644 --- a/src/core/cycle/synthesize-concepts.ts +++ b/src/core/cycle/synthesize-concepts.ts @@ -1,40 +1,241 @@ -// v0.41 T6 — synthesize_concepts cycle phase. +// v0.41 T6 — synthesize_concepts cycle phase (minimal-viable implementation). // -// SHIPPED IN T9 AS A STUB to unblock the orchestrator pack-gating test. -// The real implementation lands in T6: aggregates atoms by topic cluster, -// dedups via Jaccard + substring + semantic, tiers T1-T4 by composite_score -// (mention_count × distinct_months × breadth), Sonnet-synthesizes T1/T2 -// narratives gated by the same voice_gate() the calibration_profile phase -// uses. Concept-typed pages with tier in frontmatter. Skips pages with -// `imported_from` frontmatter marker (D7). Global scope. Budget $1.50/run. +// v0.41 ships a working concept synthesis path: group atoms by simple +// frontmatter tag/concept references, tier by count (T1 ≥10, T2 ≥5, +// T3 ≥2, T4 ≥1), Sonnet-synthesize T1/T2 narratives. Voice gate +// integration + dedup-by-embedding-similarity ship in v0.42+. // -// Until T6 ships the real body, this stub returns 'skipped' with marker. +// Sequencing: +// 1. Query all atom-typed pages from DB (excluding imported_from +// marker → atoms already extracted by wintermute don't get +// re-synthesized as concepts here; their original concept pages +// come through greenfield import already). +// 2. Group by `concepts:` frontmatter field on each atom (when the +// Haiku 3-check from extract_atoms decides "this atom is about +// concept X", it stamps the field). +// 3. For each group with count ≥2: assign tier (T1/T2/T3/T4 by count). +// 4. For T1/T2 groups: Sonnet call to produce a 1-paragraph narrative. +// For T3/T4: deterministic stub narrative. +// 5. Write concept-typed pages. import type { BrainEngine } from '../engine.ts'; import type { PhaseResult } from '../cycle.ts'; +import { chat as gatewayChat } from '../ai/gateway.ts'; + +const DEFAULT_BUDGET_USD = 1.5; +const TIER_T1_MIN = 10; +const TIER_T2_MIN = 5; +const TIER_T3_MIN = 2; export interface SynthesizeConceptsOpts { brainDir?: string; dryRun?: boolean; yieldDuringPhase?: (() => Promise) | undefined; + /** Test seam: alternative chat function. */ + _chat?: typeof gatewayChat; + /** Test seam: skip DB query; cluster these atoms directly. */ + _atoms?: Array<{ slug: string; concept_refs: string[]; body: string; title: string }>; } -/** - * v0.41 T6 stub. Same shape as runPhaseExtractAtoms stub — orchestrator - * dispatch is wired in T9, real body lands in T6. - */ +interface AtomGroup { + conceptSlug: string; + atomTitles: string[]; + atomBodies: string[]; + tier: 'T1' | 'T2' | 'T3' | 'T4'; +} + +const SYNTH_PROMPT = `You write a 1-paragraph executive summary of a concept +based on multiple atom-shaped insights that reference it. + +Output ONLY the summary paragraph (3-5 sentences). No headers, no JSON, +no preamble. Write in plain English, present-tense voice. Synthesize what +the atoms collectively SAY about the concept; don't enumerate the atoms.`; + export async function runPhaseSynthesizeConcepts( - _engine: BrainEngine, - _opts: SynthesizeConceptsOpts = {}, + engine: BrainEngine, + opts: SynthesizeConceptsOpts = {}, ): Promise { + const chat = opts._chat ?? gatewayChat; + + // 1. Get atom pages (test seam OR DB query) + let atoms = opts._atoms ?? []; + if (atoms.length === 0 && opts._atoms === undefined) { + try { + const rows = await engine.executeRaw<{ + slug: string; + title: string; + compiled_truth: string; + frontmatter: { concepts?: string[]; imported_from?: string }; + }>( + `SELECT slug, title, compiled_truth, frontmatter + FROM pages + WHERE type = 'atom' + AND deleted_at IS NULL + AND (frontmatter->>'imported_from') IS NULL`, + ); + atoms = rows + .filter((r) => Array.isArray(r.frontmatter?.concepts) && r.frontmatter.concepts.length > 0) + .map((r) => ({ + slug: r.slug, + title: r.title, + body: r.compiled_truth, + concept_refs: r.frontmatter!.concepts!, + })); + } catch { + // No atoms table or query failed — phase no-ops cleanly. + } + } + + if (atoms.length === 0) { + return { + phase: 'synthesize_concepts', + status: 'skipped', + duration_ms: 0, + summary: 'synthesize_concepts: no atoms with concept refs', + details: { reason: 'no_atoms' }, + }; + } + + // 2. Group atoms by concept slug + const groups = new Map(); + for (const atom of atoms) { + for (const conceptSlug of atom.concept_refs) { + const existing = groups.get(conceptSlug) ?? { titles: [], bodies: [] }; + existing.titles.push(atom.title); + existing.bodies.push(atom.body); + groups.set(conceptSlug, existing); + } + } + + // 3. Filter to count ≥2, assign tier + const atomGroups: AtomGroup[] = []; + for (const [conceptSlug, data] of groups) { + const count = data.titles.length; + if (count < TIER_T3_MIN) continue; + const tier: AtomGroup['tier'] = + count >= TIER_T1_MIN ? 'T1' : count >= TIER_T2_MIN ? 'T2' : 'T3'; + atomGroups.push({ + conceptSlug, + atomTitles: data.titles, + atomBodies: data.bodies, + tier, + }); + } + + if (atomGroups.length === 0) { + return { + phase: 'synthesize_concepts', + status: 'skipped', + duration_ms: 0, + summary: `synthesize_concepts: no concept groups with ≥${TIER_T3_MIN} atoms`, + details: { reason: 'no_groups_above_threshold', atoms_seen: atoms.length }, + }; + } + + // 4. Per group: synthesize narrative (LLM for T1/T2, deterministic for T3+) + let conceptsWritten = 0; + let estimatedSpendUsd = 0; + const budgetCap = DEFAULT_BUDGET_USD; + const failures: Array<{ concept: string; error: string }> = []; + const tierCounts = { T1: 0, T2: 0, T3: 0, T4: 0 }; + + for (const group of atomGroups) { + tierCounts[group.tier]++; + let narrative: string; + if (group.tier === 'T1' || group.tier === 'T2') { + if (estimatedSpendUsd >= budgetCap) { + narrative = deterministicNarrative(group); + } else { + try { + const result = await chat({ + system: SYNTH_PROMPT, + messages: [ + { + role: 'user', + content: + `Concept slug: ${group.conceptSlug}\n` + + `${group.atomTitles.length} atoms reference this concept.\n\n` + + `Sample atom titles:\n${group.atomTitles.slice(0, 10).map((t) => ` - ${t}`).join('\n')}\n\n` + + `Sample atom bodies:\n${group.atomBodies + .slice(0, 5) + .map((b, i) => `${i + 1}. ${b.slice(0, 500)}`) + .join('\n\n')}`, + }, + ], + maxTokens: 500, + }); + // Sonnet at ~$3/M input + $15/M output + estimatedSpendUsd += + (result.usage.input_tokens * 3.0 + result.usage.output_tokens * 15.0) / 1_000_000; + narrative = result.text.trim() || deterministicNarrative(group); + } catch (err) { + failures.push({ + concept: group.conceptSlug, + error: err instanceof Error ? err.message : String(err), + }); + narrative = deterministicNarrative(group); + } + } + } else { + narrative = deterministicNarrative(group); + } + + if (!opts.dryRun) { + const title = group.conceptSlug.split('/').pop() ?? group.conceptSlug; + await engine.putPage(`concepts/${title}`, { + title: title.replace(/-/g, ' '), + type: 'concept', + compiled_truth: narrative, + frontmatter: { + type: 'concept', + tier: group.tier, + mention_count: group.atomTitles.length, + composite_score: group.atomTitles.length, + synthesized_at: new Date().toISOString(), + synthesized_by: 'synthesize_concepts-v0.41', + }, + timeline: '', + }); + } + conceptsWritten++; + + if (opts.yieldDuringPhase) await opts.yieldDuringPhase(); + } + return { phase: 'synthesize_concepts', - status: 'skipped', + status: failures.length > 0 ? 'warn' : 'ok', duration_ms: 0, - summary: 'synthesize_concepts: stub (T6 not yet implemented)', + summary: + `synthesize_concepts: ${conceptsWritten} concepts ` + + `(T1=${tierCounts.T1} T2=${tierCounts.T2} T3=${tierCounts.T3})` + + (failures.length > 0 ? ` (${failures.length} LLM-failed → template fallback)` : ''), details: { - reason: 'stub_pending_t6', - note: 'orchestrator dispatch is wired; real body lands in T6', + concepts_written: conceptsWritten, + tier_counts: tierCounts, + groups_found: atomGroups.length, + atoms_seen: atoms.length, + failures, + estimated_spend_usd: estimatedSpendUsd, + budget_usd: budgetCap, + dry_run: opts.dryRun ?? false, }, }; } + +/** + * Deterministic fallback narrative for T3/T4 concepts and budget-exhausted + * T1/T2 groups. No LLM call. v0.41 minimal shape — v0.42 enriches with + * dominant themes, time spread, breadth. + */ +function deterministicNarrative(group: AtomGroup): string { + const tier = group.tier; + const count = group.atomTitles.length; + return ( + `${tier} concept. ${count} atom${count === 1 ? '' : 's'} reference this. ` + + `Top mentions:\n${group.atomTitles + .slice(0, 5) + .map((t) => ` - ${t}`) + .join('\n')}` + ); +} diff --git a/test/cycle/extract-atoms-synthesize-concepts.test.ts b/test/cycle/extract-atoms-synthesize-concepts.test.ts new file mode 100644 index 000000000..357d5a452 --- /dev/null +++ b/test/cycle/extract-atoms-synthesize-concepts.test.ts @@ -0,0 +1,284 @@ +// v0.41 T5+T6 — extract_atoms + synthesize_concepts minimal-viable bodies. +// +// Tests the LLM-driven extraction + synthesis paths with a stubbed +// chat function so no real Haiku/Sonnet calls fire in CI. Pins: +// - extract_atoms parses Haiku JSON output, writes atom-typed pages +// - parseAtomsResponse tolerates markdown fences + trailing prose +// - extract_atoms skips invalid atom_type values +// - extract_atoms budget cap halts mid-run +// - synthesize_concepts groups atoms by concept frontmatter ref +// - tier assignment by count (T1 ≥10, T2 ≥5, T3 ≥2) +// - T1/T2 use LLM narrative; T3 falls back deterministic +// - dry-run mode counts but doesn't write + +import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test'; +import { PGLiteEngine } from '../../src/core/pglite-engine.ts'; +import { runPhaseExtractAtoms, parseAtomsResponse } from '../../src/core/cycle/extract-atoms.ts'; +import { runPhaseSynthesizeConcepts } from '../../src/core/cycle/synthesize-concepts.ts'; +import { resetPgliteState } from '../helpers/reset-pglite.ts'; +import type { ChatResult, ChatOpts } from '../../src/core/ai/gateway.ts'; + +let engine: PGLiteEngine; + +beforeAll(async () => { + engine = new PGLiteEngine(); + await engine.connect({}); + await engine.initSchema(); +}, 60000); + +afterAll(async () => { + await engine.disconnect(); +}); + +beforeEach(async () => { + await resetPgliteState(engine); +}); + +function stubChat(text: string, opts: { input_tokens?: number; output_tokens?: number } = {}): (o: ChatOpts) => Promise { + return async (_o: ChatOpts) => ({ + text, + blocks: [{ type: 'text', text }], + stopReason: 'end', + usage: { + input_tokens: opts.input_tokens ?? 500, + output_tokens: opts.output_tokens ?? 200, + cache_read_tokens: 0, + cache_creation_tokens: 0, + }, + model: 'anthropic:claude-haiku-4-5', + providerId: 'anthropic', + }); +} + +describe('v0.41 T5: parseAtomsResponse', () => { + test('parses well-formed JSON array', () => { + const raw = `[{"title":"Test","atom_type":"insight","body":"body text"}]`; + const atoms = parseAtomsResponse(raw); + expect(atoms.length).toBe(1); + expect(atoms[0].title).toBe('Test'); + expect(atoms[0].atom_type).toBe('insight'); + }); + + test('strips markdown code fences', () => { + const raw = '```json\n[{"title":"T","atom_type":"quote","body":"b"}]\n```'; + expect(parseAtomsResponse(raw).length).toBe(1); + }); + + test('tolerates trailing prose after JSON', () => { + const raw = `[{"title":"T","atom_type":"framework","body":"b"}]\n\nThanks!`; + expect(parseAtomsResponse(raw).length).toBe(1); + }); + + test('rejects atoms with invalid atom_type', () => { + const raw = `[{"title":"T","atom_type":"made_up_type","body":"b"}]`; + expect(parseAtomsResponse(raw).length).toBe(0); + }); + + test('rejects atoms missing required fields', () => { + const raw = `[{"title":"T","atom_type":"insight"}]`; // no body + expect(parseAtomsResponse(raw).length).toBe(0); + }); + + test('returns [] on garbage input', () => { + expect(parseAtomsResponse('not json')).toEqual([]); + expect(parseAtomsResponse('')).toEqual([]); + }); + + test('accepts all 11 declared atom_type values', () => { + const types = ['insight', 'anecdote', 'quote', 'framework', 'statistic', + 'story_angle', 'strategy_angle', 'strategy', 'endorsement', + 'critique', 'collection']; + for (const t of types) { + const raw = `[{"title":"x","atom_type":"${t}","body":"b"}]`; + const atoms = parseAtomsResponse(raw); + expect(atoms.length).toBe(1); + expect(atoms[0].atom_type as string).toBe(t); + } + }); + + test('clamps virality_score to [0, 100]', () => { + expect(parseAtomsResponse(`[{"title":"a","atom_type":"insight","body":"b","virality_score":150}]`)[0].virality_score).toBeUndefined(); + expect(parseAtomsResponse(`[{"title":"a","atom_type":"insight","body":"b","virality_score":-5}]`)[0].virality_score).toBeUndefined(); + expect(parseAtomsResponse(`[{"title":"a","atom_type":"insight","body":"b","virality_score":75}]`)[0].virality_score).toBe(75); + }); +}); + +describe('v0.41 T5: runPhaseExtractAtoms via stubbed chat', () => { + test('no-op when no transcripts provided', async () => { + const result = await runPhaseExtractAtoms(engine, { _transcripts: [] }); + expect(result.status).toBe('skipped'); + expect(result.details?.reason).toBe('no_transcripts'); + }); + + test('extracts atoms from transcript via stub chat', async () => { + const chat = stubChat(`[ + {"title":"Renders vs physical proof","atom_type":"insight","body":"Enterprise buyers want tangible prototypes."}, + {"title":"Founder lesson","atom_type":"anecdote","body":"Story about a founder."} + ]`); + const result = await runPhaseExtractAtoms(engine, { + _transcripts: [{ filePath: '/fake/meeting.txt', content: 'content', contentHash: 'abc123def' }], + _chat: chat, + }); + expect(result.status).toBe('ok'); + expect(result.details?.atoms_extracted).toBe(2); + expect(result.details?.transcripts_processed).toBe(1); + + // Verify pages were written + const rows = await engine.executeRaw<{ slug: string; type: string }>( + `SELECT slug, type FROM pages WHERE type = 'atom'`, + ); + expect(rows.length).toBe(2); + }); + + test('dry-run counts but does NOT write', async () => { + const chat = stubChat(`[{"title":"x","atom_type":"insight","body":"b"}]`); + const result = await runPhaseExtractAtoms(engine, { + _transcripts: [{ filePath: '/x.txt', content: 'c', contentHash: 'h' }], + _chat: chat, + dryRun: true, + }); + expect(result.details?.atoms_extracted).toBe(1); + expect(result.details?.dry_run).toBe(true); + const rows = await engine.executeRaw<{ count: number }>( + `SELECT COUNT(*)::int AS count FROM pages WHERE type = 'atom'`, + ); + expect(rows[0].count).toBe(0); + }); + + test('failures tracked per-transcript without halting', async () => { + let callCount = 0; + const chat = async (_o: ChatOpts) => { + callCount++; + if (callCount === 1) throw new Error('rate limit'); + return { + text: `[{"title":"t","atom_type":"insight","body":"b"}]`, + blocks: [], + stopReason: 'end' as const, + usage: { input_tokens: 100, output_tokens: 50, cache_read_tokens: 0, cache_creation_tokens: 0 }, + model: 'anthropic:claude-haiku-4-5', + providerId: 'anthropic', + }; + }; + const result = await runPhaseExtractAtoms(engine, { + _transcripts: [ + { filePath: '/a.txt', content: 'a', contentHash: 'ha' }, + { filePath: '/b.txt', content: 'b', contentHash: 'hb' }, + ], + _chat: chat as typeof import('../../src/core/ai/gateway.ts').chat, + }); + expect(result.status).toBe('warn'); + expect(result.details?.atoms_extracted).toBe(1); + expect((result.details?.failures as unknown[]).length).toBe(1); + }); +}); + +describe('v0.41 T6: runPhaseSynthesizeConcepts via stubbed chat', () => { + test('no-op when no atoms have concept refs', async () => { + const result = await runPhaseSynthesizeConcepts(engine, { _atoms: [] }); + expect(result.status).toBe('skipped'); + expect(result.details?.reason).toBe('no_atoms'); + }); + + test('groups atoms by concept and assigns tier by count', async () => { + const atoms: Array<{ slug: string; title: string; body: string; concept_refs: string[] }> = []; + for (let i = 0; i < 12; i++) { + atoms.push({ + slug: `atoms/2026-05-24/atom-${i}`, + title: `Atom ${i}`, + body: `Body of atom ${i}.`, + concept_refs: ['ai-agents'], + }); + } + for (let i = 0; i < 6; i++) { + atoms.push({ + slug: `atoms/2026-05-24/founder-${i}`, + title: `Founder ${i}`, + body: `Founder body ${i}.`, + concept_refs: ['founder-psychology'], + }); + } + for (let i = 0; i < 3; i++) { + atoms.push({ + slug: `atoms/2026-05-24/hw-${i}`, + title: `HW ${i}`, + body: `HW body ${i}.`, + concept_refs: ['hardware-renaissance'], + }); + } + + const chat = stubChat('AI agents are software factories.'); + const result = await runPhaseSynthesizeConcepts(engine, { _atoms: atoms, _chat: chat }); + expect(result.status).toBe('ok'); + expect(result.details?.concepts_written).toBe(3); + const tiers = result.details?.tier_counts as Record; + expect(tiers.T1).toBe(1); // ai-agents (12) + expect(tiers.T2).toBe(1); // founder-psychology (6) + expect(tiers.T3).toBe(1); // hardware-renaissance (3) + }); + + test('atoms with no concept refs are filtered out', async () => { + const atoms = [ + { slug: 's1', title: 't1', body: 'b1', concept_refs: [] }, + { slug: 's2', title: 't2', body: 'b2', concept_refs: [] }, + ]; + const result = await runPhaseSynthesizeConcepts(engine, { _atoms: atoms }); + expect(result.status).toBe('skipped'); + }); + + test('concept count below T3 threshold (2) is filtered out', async () => { + const atoms = [{ slug: 's', title: 't', body: 'b', concept_refs: ['only-one-mention'] }]; + const result = await runPhaseSynthesizeConcepts(engine, { _atoms: atoms }); + expect(result.status).toBe('skipped'); + expect(result.details?.reason).toBe('no_groups_above_threshold'); + }); + + test('T3 concepts use deterministic narrative (no LLM call)', async () => { + const atoms = [ + { slug: 'a1', title: 'A1', body: 'b1', concept_refs: ['theme'] }, + { slug: 'a2', title: 'A2', body: 'b2', concept_refs: ['theme'] }, + ]; + let chatCalled = false; + const chat = async (_o: ChatOpts) => { + chatCalled = true; + return stubChat('should not be called')(_o); + }; + await runPhaseSynthesizeConcepts(engine, { _atoms: atoms, _chat: chat as typeof import('../../src/core/ai/gateway.ts').chat }); + expect(chatCalled).toBe(false); + }); + + test('dry-run counts but does NOT write', async () => { + const atoms = Array.from({ length: 6 }, (_, i) => ({ + slug: `s${i}`, + title: `T${i}`, + body: `b${i}`, + concept_refs: ['theme'], + })); + const chat = stubChat('synthesized narrative'); + const result = await runPhaseSynthesizeConcepts(engine, { + _atoms: atoms, + _chat: chat, + dryRun: true, + }); + expect(result.details?.concepts_written).toBe(1); + expect(result.details?.dry_run).toBe(true); + const rows = await engine.executeRaw<{ count: number }>( + `SELECT COUNT(*)::int AS count FROM pages WHERE type = 'concept' AND slug LIKE 'concepts/%'`, + ); + expect(rows[0].count).toBe(0); + }); + + test('T1 concept gets LLM-synthesized narrative', async () => { + const atoms = Array.from({ length: 12 }, (_, i) => ({ + slug: `a${i}`, + title: `T${i}`, + body: `b${i}`, + concept_refs: ['theme'], + })); + const chat = stubChat('Custom synthesized narrative from LLM.'); + await runPhaseSynthesizeConcepts(engine, { _atoms: atoms, _chat: chat }); + const rows = await engine.executeRaw<{ compiled_truth: string }>( + `SELECT compiled_truth FROM pages WHERE slug = 'concepts/theme'`, + ); + expect(rows[0].compiled_truth).toContain('Custom synthesized narrative'); + }); +});