diff --git a/src/core/cycle/synthesize.ts b/src/core/cycle/synthesize.ts index c6c5e8feb..c47d67f8f 100644 --- a/src/core/cycle/synthesize.ts +++ b/src/core/cycle/synthesize.ts @@ -57,6 +57,13 @@ export interface SynthesizePhaseOpts { date?: string; from?: string; to?: string; + /** + * Disable the self-consumption guard. Wired from the + * `--unsafe-bypass-dream-guard` CLI flag. NOT auto-applied for `--input` + * because that would allow any dream-generated page to silently re-enter + * the synthesize loop. Caller must opt in explicitly. + */ + bypassDreamGuard?: boolean; } export async function runPhaseSynthesize( @@ -87,9 +94,16 @@ export async function runPhaseSynthesize( } } + if (opts.bypassDreamGuard) { + process.stderr.write( + '[dream] WARNING: --unsafe-bypass-dream-guard set; self-consumption guard disabled. ' + + 'Re-ingestion of dream output will incur Sonnet costs forever.\n', + ); + } + // Discover. const transcripts = opts.inputFile - ? loadAdHocTranscript(opts.inputFile, config.minChars, config.excludePatterns) + ? loadAdHocTranscript(opts.inputFile, config.minChars, config.excludePatterns, opts.bypassDreamGuard) : discoverTranscripts({ corpusDir: config.corpusDir!, meetingTranscriptsDir: config.meetingTranscriptsDir ?? undefined, @@ -98,6 +112,7 @@ export async function runPhaseSynthesize( date: opts.date, from: opts.from, to: opts.to, + bypassGuard: opts.bypassDreamGuard, }); if (transcripts.length === 0) { @@ -321,7 +336,7 @@ async function loadAllowedSlugPrefixes(): Promise { // ── Significance judge (Haiku) ─────────────────────────────────────── -interface JudgeClient { +export interface JudgeClient { create: (params: Anthropic.MessageCreateParamsNonStreaming) => Promise; } @@ -336,7 +351,7 @@ interface VerdictResult { reasons: string[]; } -async function judgeSignificance( +export async function judgeSignificance( client: JudgeClient, t: DiscoveredTranscript, verdictModel = 'claude-haiku-4-5-20251001', @@ -484,10 +499,20 @@ async function reverseWriteSlugs( return count; } -function renderPageToMarkdown(page: Page, tags: string[]): string { - // serializeMarkdown's contract: takes (frontmatter, compiled_truth, timeline, meta) - // and emits frontmatter + body + (optional) timeline section. - const frontmatter = (page.frontmatter ?? {}) as Record; +/** + * Render a Page to markdown, stamping the dream-output identity marker into + * frontmatter. This stamp is the explicit identity surface checked by + * `isDreamOutput` in transcript-discovery.ts. Stamping at render time covers + * every reverse-write path (subagent reflections + originals + summary) with + * one funnel; the prior content-pattern guard could miss real output because + * `serializeMarkdown` does not embed the page slug in the body. + */ +export function renderPageToMarkdown(page: Page, tags: string[]): string { + const frontmatter: Record = { + ...((page.frontmatter ?? {}) as Record), + dream_generated: true, + dream_cycle_date: today(), + }; return serializeMarkdown( frontmatter, page.compiled_truth ?? '', @@ -529,8 +554,11 @@ async function writeSummaryPage( } const body = lines.join('\n'); + // Stamp the dream-output identity marker into the summary's frontmatter. + // parseMarkdown below round-trips it into the DB-stored frontmatter, so the + // marker survives any later reverse-render of the summary page. const fullMarkdown = serializeMarkdown( - {} as Record, + { dream_generated: true, dream_cycle_date: summaryDate } as Record, body, '', { type: 'note' as PageType, title: `Dream cycle ${summaryDate}`, tags: ['dream-cycle'] }, @@ -568,9 +596,10 @@ function loadAdHocTranscript( filePath: string, minChars: number, excludePatterns: string[], + bypassGuard?: boolean, ): DiscoveredTranscript[] { const { readSingleTranscript } = require('./transcript-discovery.ts') as typeof import('./transcript-discovery.ts'); - const t = readSingleTranscript(filePath, { minChars, excludePatterns }); + const t = readSingleTranscript(filePath, { minChars, excludePatterns, bypassGuard }); return t ? [t] : []; } diff --git a/src/core/cycle/transcript-discovery.ts b/src/core/cycle/transcript-discovery.ts index d1ee0cd81..19ae56ba5 100644 --- a/src/core/cycle/transcript-discovery.ts +++ b/src/core/cycle/transcript-discovery.ts @@ -41,30 +41,39 @@ export interface DiscoverOpts { from?: string; /** Inclusive range end (YYYY-MM-DD). */ to?: string; + /** + * Disable the self-consumption guard. Caller must opt in explicitly via + * `--unsafe-bypass-dream-guard`; never auto-applied for `--input` because + * that would let any caller silently re-trigger the loop bug. + */ + bypassGuard?: boolean; } const DATE_RE = /^(\d{4}-\d{2}-\d{2})/; const WORD_BOUNDARY_HEURISTIC = /^[a-zA-Z][a-zA-Z0-9_-]*$/; /** - * Built-in self-consumption guard: slug prefixes that indicate dream output. - * If a transcript contains any of these paths, it's the dream cycle's own - * output being fed back in — skip it to prevent infinite recursion. - * This is a hard-coded safety net; user-configured exclude_patterns are - * additive on top of this. + * Self-consumption guard: identity-marker check against `dream_generated: true` + * stamped by the synthesize phase's render paths. + * + * v0.23.1 used a body slug-prefix string match. Codex review of the v0.23.2 + * plan caught two flaws: (1) `serializeMarkdown` does NOT embed the page slug + * into body content, so the prefix heuristic could miss real dream output, and + * (2) real conversation transcripts that legitimately cite a brain page would + * be silently dropped. v0.23.2 swaps content inference for explicit identity + * stamped at render time. + * + * Regex anchored at frontmatter open (`---\n`), tolerates optional BOM and CRLF, + * scans the first 2000 chars for `dream_generated: true` (any whitespace, case- + * insensitive value, word boundary on `true`). */ -const DREAM_OUTPUT_SLUGS = [ - 'wiki/personal/reflections/', - 'wiki/originals/ideas/', - 'wiki/personal/patterns/', - 'dream-cycle-summaries/', -]; +const DREAM_MARKER_REGEX_SRC = + '^\\uFEFF?-{3}\\r?\\n[\\s\\S]{0,2000}?dream_generated\\s*:\\s*true\\b'; +export const DREAM_OUTPUT_MARKER_RE = new RegExp(DREAM_MARKER_REGEX_SRC, 'i'); -function isDreamOutput(content: string): boolean { - // Check the first 2000 chars (frontmatter + opening) for dream output markers. - // Full-content scan is wasteful; dream output always self-identifies early. - const head = content.slice(0, 2000); - return DREAM_OUTPUT_SLUGS.some(slug => head.includes(slug)); +export function isDreamOutput(content: string, bypass = false): boolean { + if (bypass) return false; + return DREAM_OUTPUT_MARKER_RE.test(content); } /** @@ -136,12 +145,14 @@ function listTextFiles(dir: string): string[] { * - aren't `.txt` * - have date-prefixed basenames outside the requested window * - have content shorter than `minChars` + * - carry the `dream_generated: true` self-consumption marker (unless `bypassGuard`) * - match any compiled exclude pattern (case-insensitive word-boundary by default) * * Returns sorted by filePath so re-runs are deterministic. */ export function discoverTranscripts(opts: DiscoverOpts): DiscoveredTranscript[] { const minChars = opts.minChars ?? 2000; + const bypass = opts.bypassGuard === true; const excludeRes = compileExcludePatterns(opts.excludePatterns); const dirs = [opts.corpusDir, opts.meetingTranscriptsDir].filter( (d): d is string => typeof d === 'string' && d.length > 0, @@ -162,7 +173,10 @@ export function discoverTranscripts(opts: DiscoverOpts): DiscoveredTranscript[] continue; } if (content.length < minChars) continue; - if (isDreamOutput(content)) continue; + if (isDreamOutput(content, bypass)) { + process.stderr.write(`[dream] skipped ${baseName}: dream_generated marker (self-consumption guard)\n`); + continue; + } if (matchesAnyExclude(content, excludeRes)) continue; results.push({ @@ -181,13 +195,15 @@ export function discoverTranscripts(opts: DiscoverOpts): DiscoveredTranscript[] /** * Read a single ad-hoc transcript file (`gbrain dream --input `). * Bypasses the corpus-dir scan and date filters but still applies - * minChars + exclude_patterns when provided. + * minChars + exclude_patterns when provided. The self-consumption guard + * also still fires unless `bypassGuard` is set explicitly. */ export function readSingleTranscript( filePath: string, - opts: { minChars?: number; excludePatterns?: string[] } = {}, + opts: { minChars?: number; excludePatterns?: string[]; bypassGuard?: boolean } = {}, ): DiscoveredTranscript | null { const minChars = opts.minChars ?? 2000; + const bypass = opts.bypassGuard === true; const excludeRes = compileExcludePatterns(opts.excludePatterns); let content: string; try { @@ -197,7 +213,11 @@ export function readSingleTranscript( throw new Error(`could not read transcript at ${filePath}: ${msg}`); } if (content.length < minChars) return null; - if (isDreamOutput(content)) return null; + if (isDreamOutput(content, bypass)) { + const baseName = basename(filePath, '.txt'); + process.stderr.write(`[dream] readSingleTranscript skipped ${baseName}: dream_generated marker (self-consumption guard)\n`); + return null; + } if (matchesAnyExclude(content, excludeRes)) return null; const baseName = basename(filePath, '.txt'); const dateMatch = DATE_RE.exec(baseName); diff --git a/test/cycle-synthesize.test.ts b/test/cycle-synthesize.test.ts index 0aa7c1ded..cfee8c6a0 100644 --- a/test/cycle-synthesize.test.ts +++ b/test/cycle-synthesize.test.ts @@ -15,7 +15,10 @@ import { discoverTranscripts, readSingleTranscript, compileExcludePatterns, + isDreamOutput, + DREAM_OUTPUT_MARKER_RE, } from '../src/core/cycle/transcript-discovery.ts'; +import { judgeSignificance, renderPageToMarkdown, type JudgeClient } from '../src/core/cycle/synthesize.ts'; let tmpDir: string; @@ -190,35 +193,136 @@ describe('readSingleTranscript', () => { }); }); -describe('self-consumption guard', () => { - test('discoverTranscripts skips files containing dream output slug prefixes', () => { - makeTranscript('2026-04-25-real.txt', 'This is a real conversation transcript. ' + 'x'.repeat(3000)); - makeTranscript('2026-04-25-dream.txt', - '---\ntitle: Reflection\n---\n# Some reflection\nwiki/personal/reflections/2026-04-25-foo-abc123\n' + 'x'.repeat(3000)); - makeTranscript('2026-04-25-original.txt', - '---\ntitle: Idea\n---\n# Some idea\nwiki/originals/ideas/2026-04-25-bar-def456\n' + 'x'.repeat(3000)); - makeTranscript('2026-04-25-pattern.txt', - '---\ntype: pattern\n---\nwiki/personal/patterns/shame-cycle\n' + 'x'.repeat(3000)); - makeTranscript('2026-04-25-summary.txt', - 'dream-cycle-summaries/2026-04-25 generated this page\n' + 'x'.repeat(3000)); - - const results = discoverTranscripts({ corpusDir: tmpDir, minChars: 1000 }); - expect(results).toHaveLength(1); - expect(results[0].basename).toBe('2026-04-25-real'); - }); - - test('readSingleTranscript returns null for dream output', () => { - const path = makeTranscript('2026-04-25-dream.txt', - 'wiki/personal/reflections/2026-04-25-thing-abc123\n' + 'x'.repeat(3000)); - const result = readSingleTranscript(path, { minChars: 1000 }); +describe('self-consumption guard (v0.23.2 marker-based)', () => { + test('REGRESSION: catches actual reverseWriteSlugs output from a real Page', () => { + // Build a Page like the synthesize subagent would produce, run it through + // the same renderPageToMarkdown the orchestrator uses, and assert the guard + // fires. Codex finding #5: synthetic-string fixtures don't prove the guard + // catches what the synthesize phase actually produces. + const page = { + slug: 'wiki/personal/reflections/2026-04-30-test-abc123', + type: 'reflection' as const, + title: 'Test reflection', + compiled_truth: 'I learned something about [Alice](people/alice). No own-slug citation in body.', + timeline: '', + frontmatter: {}, + }; + const md = renderPageToMarkdown(page as any, ['dream-cycle']); + const path = makeTranscript('2026-04-30-output.txt', md + '\n' + 'x'.repeat(3000)); + const result = readSingleTranscript(path, { minChars: 100 }); expect(result).toBeNull(); }); - test('self-consumption guard checks only first 2000 chars (performance)', () => { - // Dream slug buried deep in the file should NOT trigger the guard - const content = 'x'.repeat(3000) + 'wiki/personal/reflections/2026-04-25-buried-abc123'; - const path = makeTranscript('2026-04-25-deep.txt', content); - const result = readSingleTranscript(path, { minChars: 1000 }); + test('does NOT fire on real conversation transcript citing a brain slug', () => { + // The exact false-positive case codex finding #1 named: a user note that + // legitimately mentions a reflection slug in plain text. Must NOT be skipped. + const path = makeTranscript('convo.txt', + 'User: tell me about wiki/personal/reflections/identity-foo and how it relates to my work.\n' + + 'Agent: ' + 'x'.repeat(3000)); + const result = readSingleTranscript(path, { minChars: 100 }); expect(result).not.toBeNull(); }); + + test('CRLF + BOM frontmatter still triggers guard', () => { + const content = '\uFEFF---\r\ndream_generated: true\r\n---\r\n# x\r\n' + 'x'.repeat(3000); + const path = makeTranscript('crlf.txt', content); + const result = readSingleTranscript(path, { minChars: 100 }); + expect(result).toBeNull(); + }); + + test('whitespace and case tolerance: matches dream_generated: true variants', () => { + const variants = [ + '---\ndream_generated:true\n---\nbody' + 'x'.repeat(3000), + '---\ndream_generated: true\n---\nbody' + 'x'.repeat(3000), + '---\ndream_generated: TRUE\n---\nbody' + 'x'.repeat(3000), + '---\ntitle: foo\ndream_generated: true\n---\nbody' + 'x'.repeat(3000), + ]; + for (const variant of variants) { + expect(isDreamOutput(variant)).toBe(true); + } + }); + + test('does NOT fire when dream_generated is false or absent', () => { + expect(isDreamOutput('---\ntitle: foo\n---\nbody')).toBe(false); + expect(isDreamOutput('---\ndream_generated: false\n---\nbody')).toBe(false); + expect(isDreamOutput('plain text with no frontmatter')).toBe(false); + // dream_generatedfoo: true (no word boundary on the key) must NOT match + expect(isDreamOutput('---\ndream_generatedfoo: true\n---\nbody')).toBe(false); + }); + + test('marker buried past 2000 chars does NOT trigger guard (perf bound)', () => { + const padding = 'x'.repeat(2100); + const content = '---\ntitle: real\n---\n' + padding + '\ndream_generated: true\n' + 'x'.repeat(3000); + const path = makeTranscript('buried.txt', content); + const result = readSingleTranscript(path, { minChars: 100 }); + expect(result).not.toBeNull(); + }); + + test('bypassGuard=true overrides marker (--unsafe-bypass-dream-guard plumbing)', () => { + const md = '---\ndream_generated: true\n---\n# Page\n' + 'x'.repeat(3000); + const path = makeTranscript('marked.txt', md); + expect(readSingleTranscript(path, { minChars: 100 })).toBeNull(); + expect(readSingleTranscript(path, { minChars: 100, bypassGuard: true })).not.toBeNull(); + }); + + test('discoverTranscripts respects bypassGuard', () => { + const md = '---\ndream_generated: true\n---\n# Page\n' + 'x'.repeat(3000); + makeTranscript('2026-04-30-output.txt', md); + makeTranscript('2026-04-30-real.txt', 'real transcript ' + 'x'.repeat(3000)); + + const guarded = discoverTranscripts({ corpusDir: tmpDir, minChars: 100 }); + expect(guarded).toHaveLength(1); + expect(guarded[0].basename).toBe('2026-04-30-real'); + + const bypassed = discoverTranscripts({ corpusDir: tmpDir, minChars: 100, bypassGuard: true }); + expect(bypassed).toHaveLength(2); + }); + + test('DREAM_OUTPUT_MARKER_RE is anchored at file start (not mid-content)', () => { + // Frontmatter delimiter must be at byte 0; mid-content `---\n` does not count. + const content = 'preamble\n---\ndream_generated: true\n---\nbody' + 'x'.repeat(3000); + expect(DREAM_OUTPUT_MARKER_RE.test(content)).toBe(false); + }); +}); + +describe('judgeSignificance', () => { + function makeTranscript(): import('../src/core/cycle/transcript-discovery.ts').DiscoveredTranscript { + return { + filePath: '/tmp/x.txt', + contentHash: 'abc123', + content: 'A short conversation about something interesting.', + basename: 'x', + inferredDate: null, + }; + } + + function mockClient(captured: { model?: string }): JudgeClient { + return { + create: async (p: any) => { + captured.model = p.model; + return { content: [{ type: 'text', text: '{"worth_processing": true, "reasons": ["test"]}' }] } as any; + }, + }; + } + + test('passes verdict_model override to client.create', async () => { + const captured: { model?: string } = {}; + await judgeSignificance(mockClient(captured), makeTranscript(), 'claude-sonnet-4-6'); + expect(captured.model).toBe('claude-sonnet-4-6'); + }); + + test('defaults to claude-haiku-4-5-20251001 when model omitted', async () => { + const captured: { model?: string } = {}; + await judgeSignificance(mockClient(captured), makeTranscript()); + expect(captured.model).toBe('claude-haiku-4-5-20251001'); + }); + + test('returns worth_processing=false when judge returns unparseable text', async () => { + const client: JudgeClient = { + create: async () => ({ content: [{ type: 'text', text: 'no json here' }] } as any), + }; + const r = await judgeSignificance(client, makeTranscript()); + expect(r.worth_processing).toBe(false); + expect(r.reasons[0]).toContain('unparseable'); + }); });