fix(doctor): raw-source persistence guarantee for synthesized pages — warn-only v1 (#3300)

* feat(doctor): raw-source persistence guarantee — warn-only v1 (#1978)

Every synthesized/derived page (dream_generated:true or type:synthesis)
must carry a raw trace or an explicit exemption. v1 is warn-only:

- New doctor check `raw_provenance` (brain category) flags synthesized
  pages with none of: raw_trace/raw_source/source_uri/raw_trace_exempt
  frontmatter, an attached raw_data row, or synthesis_evidence rows.
- Dream synthesize now stamps `raw_source: <transcript path>` into each
  written page's frontmatter via the existing #2569 provenance stamp.
- Dream-cycle summary index pages and extract receipts carry an explicit
  `raw_trace_exempt: true` + reason (no source document of their own).

No write path is blocked; fail-closed enforcement is the v2 escalation.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(doctor): exclude soft-deleted pages from raw_provenance check

Sibling frontmatter checks (quarantined_pages, flagged_pages) filter
deleted_at IS NULL; without it a deleted synthesized page keeps warning
(and its slug keeps being named) through the 72h recovery window with
no way to clear the warn.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: Garry Tan <garrytan@gmail.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

src/commands/doctor.ts

@@ -529,6 +529,61 @@ export async function childTableOrphansCheck(engine: BrainEngine): Promise<Check
   };
 }
 
+/**
+ * Raw-source persistence guarantee (#1978, warn-only v1).
+ *
+ * Invariant: every synthesized/derived page (dream_generated:true frontmatter
+ * or type:synthesis) must either carry a raw trace or declare an explicit
+ * exemption. Accepted traces:
+ *   - frontmatter key `raw_trace` / `raw_source` / `source_uri`
+ *   - an attached `raw_data` row
+ *   - `synthesis_evidence` rows (think-op citations)
+ *   - explicit `raw_trace_exempt: true` (reason in `raw_trace_exempt_reason`)
+ *
+ * v1 is deliberately warn-only — no write path is blocked. Escalation to
+ * fail-closed enforcement in the synthesis/import write paths is the v2
+ * follow-up once real brains run clean.
+ *
+ * Pure helper (engine.executeRaw only) for parity with
+ * childTableOrphansCheck so tests can target it directly.
+ */
+export async function rawProvenanceCheck(engine: BrainEngine): Promise<Check> {
+  const where = `
+        p.deleted_at IS NULL
+    AND (COALESCE(p.frontmatter->>'dream_generated', '') = 'true' OR p.type = 'synthesis')
+    AND NOT (COALESCE(p.frontmatter, '{}'::jsonb) ?| ARRAY['raw_trace', 'raw_source', 'source_uri', 'raw_trace_exempt'])
+    AND NOT EXISTS (SELECT 1 FROM raw_data rd WHERE rd.page_id = p.id)
+    AND NOT EXISTS (SELECT 1 FROM synthesis_evidence se WHERE se.synthesis_page_id = p.id)`;
+  try {
+    const rows = await engine.executeRaw<{ n: string | number }>(
+      `SELECT COUNT(*)::int AS n FROM pages p WHERE ${where}`,
+    );
+    const n = Number(rows[0]?.n ?? 0);
+    if (n === 0) {
+      return {
+        name: 'raw_provenance',
+        status: 'ok',
+        message: 'All synthesized pages carry a raw trace or explicit exemption',
+      };
+    }
+    const sample = await engine.executeRaw<{ slug: string }>(
+      `SELECT p.slug FROM pages p WHERE ${where} ORDER BY p.slug LIMIT 5`,
+    );
+    const slugs = sample.map(r => r.slug).join(', ');
+    return {
+      name: 'raw_provenance',
+      status: 'warn',
+      message:
+        `${n} synthesized page(s) lack a raw trace (no raw_trace/raw_source/source_uri frontmatter, ` +
+        `raw_data row, or synthesis evidence) and carry no raw_trace_exempt marker. e.g. ${slugs}. ` +
+        `Fix: stamp raw_source (path/URI of the source material) or raw_trace_exempt: true + ` +
+        `raw_trace_exempt_reason in frontmatter. Warn-only (#1978).`,
+    };
+  } catch {
+    return { name: 'raw_provenance', status: 'warn', message: 'Could not check raw provenance (older schema?)' };
+  }
+}
+
 export async function doctorReportRemote(engine: BrainEngine): Promise<DoctorReport> {
   const checks: Check[] = [];
 
@@ -6290,6 +6345,12 @@ export async function buildChecks(
   progress.heartbeat('child_table_orphans');
   checks.push(await childTableOrphansCheck(engine));
 
+  // 10d. Raw-source persistence guarantee (#1978, warn-only v1).
+  // Every synthesized/derived page must carry a raw trace or an explicit
+  // exemption. Warn-only in v1 — surfaces violations, blocks nothing.
+  progress.heartbeat('raw_provenance');
+  checks.push(await rawProvenanceCheck(engine));
+
   // v0.33: whoknows_health — fixture presence + row count. The eval
   // gate itself runs via `gbrain eval whoknows`; this check is the
   // "did you do the assignment?" signal.

src/core/cycle/synthesize.ts

@@ -554,6 +554,8 @@ export async function runPhaseSynthesize(
     const childIds: number[] = [];
     /** Map child job_id → chunk metadata for D6 orchestrator-side slug rewrite. */
     const chunkInfo = new Map<number, { idx: number; hash6: string }>();
+    /** #1978: map child job_id → source transcript path so written pages get a raw_source stamp. */
+    const jobRawSource = new Map<number, string>();
     /** Skip reasons for the cycle report (D5 cap hits, D8 legacy-key skips). */
     const skipReports: Array<{ filePath: string; reason: string }> = [];
 
@@ -638,6 +640,7 @@ export async function runPhaseSynthesize(
           { allowProtectedSubmit: true },
         );
         childIds.push(child.id);
+        jobRawSource.set(child.id, t.filePath);
         if (isChunked) {
           chunkInfo.set(child.id, { idx: i, hash6 });
         }
@@ -682,7 +685,7 @@ export async function runPhaseSynthesize(
     // (source, slug) row. #1586: refs are stamped with the cycle's resolved
     // source (children write there via SubagentHandlerData.source_id).
     const cycleSourceId = opts.sourceId ?? 'default';
-    const writtenRefs = await collectChildPutPageSlugs(engine, childIds, chunkInfo, cycleSourceId);
+    const writtenRefs = await collectChildPutPageSlugs(engine, childIds, chunkInfo, cycleSourceId, jobRawSource);
 
     const summaryDate = opts.date ?? today();
 
@@ -1234,7 +1237,8 @@ async function collectChildPutPageSlugs(
   childIds: number[],
   chunkInfo: Map<number, { idx: number; hash6: string }>,
   sourceId = 'default',
-): Promise<Array<{ slug: string; source_id: string }>> {
+  jobRawSource?: Map<number, string>,
+): Promise<Array<{ slug: string; source_id: string; raw_source?: string }>> {
   if (childIds.length === 0) return [];
   // Raw fetch — NO SELECT DISTINCT. Preserves per-child slug duplicates so
   // the orchestrator sees what each child wrote. COALESCE handles both
@@ -1256,13 +1260,21 @@ async function collectChildPutPageSlugs(
         AND status = 'complete'`,
     [childIds],
   );
-  const rewritten = new Set<string>();
+  // #1978: slug → source transcript path (first writer wins) so the
+  // provenance stamp can record WHERE the synthesized content came from.
+  const rewritten = new Map<string, string | undefined>();
   for (const r of rows) {
     if (typeof r.slug !== 'string' || r.slug.length === 0) continue;
     const ci = chunkInfo.get(r.job_id);
-    rewritten.add(ci ? rewriteChunkedSlug(r.slug, ci.hash6, ci.idx) : r.slug);
+    const slug = ci ? rewriteChunkedSlug(r.slug, ci.hash6, ci.idx) : r.slug;
+    if (!rewritten.has(slug) || rewritten.get(slug) === undefined) {
+      rewritten.set(slug, jobRawSource?.get(r.job_id));
+    }
   }
-  return Array.from(rewritten).sort().map(slug => ({ slug, source_id: sourceId }));
+  return Array.from(rewritten.keys()).sort().map(slug => {
+    const raw_source = rewritten.get(slug);
+    return { slug, source_id: sourceId, ...(raw_source ? { raw_source } : {}) };
+  });
 }
 
 /**
@@ -1308,12 +1320,12 @@ async function hasLegacySingleChunkCompletion(
  */
 async function stampDreamProvenance(
   engine: BrainEngine,
-  refs: Array<{ slug: string; source_id: string }>,
+  refs: Array<{ slug: string; source_id: string; raw_source?: string }>,
   cycleDate: string,
 ): Promise<void> {
   if (refs.length === 0) return;
   const { executeRawJsonb } = await import('../sql-query.ts');
-  for (const { slug, source_id } of refs) {
+  for (const { slug, source_id, raw_source } of refs) {
     try {
       await executeRawJsonb(
         engine,
@@ -1321,7 +1333,14 @@ async function stampDreamProvenance(
             SET frontmatter = COALESCE(frontmatter, '{}'::jsonb) || $3::jsonb
           WHERE slug = $1 AND source_id = $2`,
         [slug, source_id],
-        [{ dream_generated: true, dream_cycle_date: cycleDate }],
+        // #1978 raw-source persistence: record the transcript path the
+        // synthesis was derived from, so `gbrain doctor` (raw_provenance
+        // check) can verify every generated page carries a raw trace.
+        [{
+          dream_generated: true,
+          dream_cycle_date: cycleDate,
+          ...(raw_source ? { raw_source } : {}),
+        }],
       );
     } catch (e) {
       const msg = e instanceof Error ? e.message : String(e);
@@ -1423,7 +1442,15 @@ async function writeSummaryPage(
   // parseMarkdown below round-trips it into the DB-stored frontmatter, so the
   // marker survives any later reverse-render of the summary page.
   const fullMarkdown = serializeMarkdown(
-    { dream_generated: true, dream_cycle_date: summaryDate } as Record<string, unknown>,
+    {
+      dream_generated: true,
+      dream_cycle_date: summaryDate,
+      // #1978: deterministic index page — no source document of its own;
+      // raw traces live on the listed pages. Explicit exemption keeps the
+      // doctor raw_provenance check quiet.
+      raw_trace_exempt: true,
+      raw_trace_exempt_reason: 'deterministic dream-cycle index; raw traces live on listed pages',
+    } as Record<string, unknown>,
     body,
     '',
     { type: 'note' as string, title: `Dream cycle ${summaryDate}`, tags: ['dream-cycle'] },

src/core/doctor-categories.ts

@@ -99,6 +99,7 @@ export const BRAIN_CHECK_NAMES: ReadonlySet<string> = new Set([
   'orphan_ratio',
   'oversized_pages',
   'quarantined_pages',
+  'raw_provenance',
   'flagged_pages',
   'salience_health',
   'scraper_junk_pages',

src/core/extract/receipt-writer.ts

@@ -157,6 +157,11 @@ function buildReceiptFrontmatter(input: ExtractReceiptInput): Record<string, unk
   const fm: Record<string, unknown> = {
     type: 'extract_receipt',
     dream_generated: true,
+    // #1978: receipts record an operation, not a source document — the
+    // run_id/round fields ARE the provenance. Explicit exemption keeps the
+    // doctor raw_provenance check quiet.
+    raw_trace_exempt: true,
+    raw_trace_exempt_reason: 'operation receipt; provenance is run_id + round',
     kind: input.kind,
     source_id: input.source_id,
     run_id: input.run_id,

test/cycle-synthesize-slug-collection.test.ts

@@ -117,6 +117,22 @@ describe('C6: collectChildPutPageSlugs survives double-encoded jsonb (#745)', ()
     expect(refs.length).toBeGreaterThan(0);
     for (const r of refs) expect(r.source_id).toBe('default');
   });
+
+  // #1978: refs carry the source transcript path when the orchestrator
+  // supplies a job_id → path map, so stampDreamProvenance can persist it.
+  test('stamps refs with raw_source from the jobRawSource map (#1978)', async () => {
+    const jobRawSource = new Map([[1001, '/transcripts/2026-07-01-standup.md']]);
+    const refs = await collectChildPutPageSlugs(engine as any, [1001], new Map(), 'default', jobRawSource);
+    const ref = refs.find((r: { slug: string }) => r.slug === 'wiki/agents/test/normal-shape');
+    expect(ref?.raw_source).toBe('/transcripts/2026-07-01-standup.md');
+  });
+
+  test('omits raw_source when no map entry exists for the job (#1978)', async () => {
+    const refs = await collectChildPutPageSlugs(engine as any, [1001], new Map(), 'default', new Map());
+    const ref = refs.find((r: { slug: string }) => r.slug === 'wiki/agents/test/normal-shape');
+    expect(ref).toBeDefined();
+    expect('raw_source' in (ref as object)).toBe(false);
+  });
 });
 
 describe('#2569: stampDreamProvenance persists the marker into DB frontmatter', () => {
@@ -151,4 +167,31 @@ describe('#2569: stampDreamProvenance persists the marker into DB frontmatter',
     await stampDreamProvenance(engine as any, refs, '2026-07-17'); // no throw
     await stampDreamProvenance(engine as any, refs, '2026-07-17'); // idempotent
   });
+
+  // #1978: raw-source persistence — the stamp carries the transcript path
+  // the synthesis was derived from, when the ref supplies one.
+  test('persists raw_source into pages.frontmatter when the ref carries it (#1978)', async () => {
+    await engine.putPage('wiki/originals/ideas/2026-07-17-raw-src-def456', {
+      type: 'note',
+      title: 'Raw source stamp',
+      compiled_truth: 'body',
+      timeline: '',
+      frontmatter: {},
+    });
+    await stampDreamProvenance(
+      engine as any,
+      [{
+        slug: 'wiki/originals/ideas/2026-07-17-raw-src-def456',
+        source_id: 'default',
+        raw_source: '/transcripts/2026-07-17-standup.md',
+      }],
+      '2026-07-17',
+    );
+    const rows = await engine.executeRaw<{ fm: Record<string, unknown> }>(
+      `SELECT frontmatter AS fm FROM pages WHERE slug = 'wiki/originals/ideas/2026-07-17-raw-src-def456'`,
+    );
+    const fm = rows[0].fm as Record<string, unknown>;
+    expect(fm.dream_generated).toBe(true);
+    expect(fm.raw_source).toBe('/transcripts/2026-07-17-standup.md');
+  });
 });

test/doctor-raw-provenance.test.ts

@@ -0,0 +1,110 @@
+/**
+ * #1978 — raw-source persistence guarantee (warn-only v1).
+ *
+ * `rawProvenanceCheck` flags synthesized/derived pages (dream_generated:true
+ * frontmatter or type:synthesis) that carry NO raw trace (raw_trace /
+ * raw_source / source_uri frontmatter, attached raw_data row, or
+ * synthesis_evidence rows) and NO explicit raw_trace_exempt marker.
+ *
+ * Runs against real PGLite so the SQL shape (`?|` key-existence operator +
+ * NOT EXISTS subqueries) is pinned on an actual engine, not a mock.
+ */
+
+import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
+import { PGLiteEngine } from '../src/core/pglite-engine.ts';
+import { rawProvenanceCheck } from '../src/commands/doctor.ts';
+import { categorizeCheck } from '../src/core/doctor-categories.ts';
+import type { BrainEngine } from '../src/core/engine.ts';
+
+let engine: PGLiteEngine;
+
+beforeAll(async () => {
+  engine = new PGLiteEngine();
+  await engine.connect({});
+  await engine.initSchema();
+});
+
+afterAll(async () => {
+  await engine.disconnect();
+});
+
+describe('rawProvenanceCheck (#1978, warn-only v1)', () => {
+  test('empty brain → ok', async () => {
+    const result = await rawProvenanceCheck(engine as unknown as BrainEngine);
+    expect(result.name).toBe('raw_provenance');
+    expect(result.status).toBe('ok');
+  });
+
+  test('flags only the synthesized page without a trace; every trace/exemption shape passes', async () => {
+    // 1. VIOLATION: dream-generated, no trace, no exemption.
+    await engine.putPage('wiki/derived/no-trace', {
+      type: 'note', title: 'No trace', compiled_truth: 'body', timeline: '',
+      frontmatter: { dream_generated: true },
+    });
+    // 2. OK: dream-generated with raw_source frontmatter.
+    await engine.putPage('wiki/derived/with-raw-source', {
+      type: 'note', title: 'Has raw_source', compiled_truth: 'body', timeline: '',
+      frontmatter: { dream_generated: true, raw_source: '/transcripts/2026-07-01.md' },
+    });
+    // 3. OK: type:synthesis with explicit exemption.
+    await engine.putPage('synthesis/exempt-page', {
+      type: 'synthesis', title: 'Exempt', compiled_truth: 'body', timeline: '',
+      frontmatter: { raw_trace_exempt: true, raw_trace_exempt_reason: 'test' },
+    });
+    // 4. OK: hand-authored note — not synthesized, never flagged.
+    await engine.putPage('wiki/hand-authored', {
+      type: 'note', title: 'Hand authored', compiled_truth: 'body', timeline: '',
+      frontmatter: {},
+    });
+    // 5. OK: dream-generated with an attached raw_data row.
+    const withRaw = await engine.putPage('wiki/derived/with-raw-data', {
+      type: 'note', title: 'Has raw_data', compiled_truth: 'body', timeline: '',
+      frontmatter: { dream_generated: true },
+    });
+    await engine.executeRaw(
+      `INSERT INTO raw_data (page_id, source, data) VALUES ($1, 'test', '{}'::jsonb)`,
+      [withRaw.id],
+    );
+
+    const result = await rawProvenanceCheck(engine as unknown as BrainEngine);
+    expect(result.status).toBe('warn');
+    expect(result.message).toContain('1 synthesized page(s)');
+    expect(result.message).toContain('wiki/derived/no-trace');
+    expect(result.message).not.toContain('with-raw-source');
+    expect(result.message).not.toContain('exempt-page');
+    expect(result.message).not.toContain('hand-authored');
+    expect(result.message).not.toContain('with-raw-data');
+  });
+
+  test('stamping an exemption on the violator clears the warning', async () => {
+    await engine.executeRaw(
+      `UPDATE pages SET frontmatter = frontmatter || '{"raw_trace_exempt": true, "raw_trace_exempt_reason": "reviewed"}'::jsonb
+        WHERE slug = 'wiki/derived/no-trace'`,
+    );
+    const result = await rawProvenanceCheck(engine as unknown as BrainEngine);
+    expect(result.status).toBe('ok');
+  });
+
+  test('soft-deleted violators are not flagged', async () => {
+    await engine.putPage('wiki/derived/deleted-no-trace', {
+      type: 'note', title: 'Deleted violator', compiled_truth: 'body', timeline: '',
+      frontmatter: { dream_generated: true },
+    });
+    expect((await rawProvenanceCheck(engine as unknown as BrainEngine)).status).toBe('warn');
+    await engine.executeRaw(
+      `UPDATE pages SET deleted_at = now() WHERE slug = 'wiki/derived/deleted-no-trace'`,
+    );
+    expect((await rawProvenanceCheck(engine as unknown as BrainEngine)).status).toBe('ok');
+  });
+
+  test('query failure degrades to warn, never throws', async () => {
+    const broken = { executeRaw: async () => { throw new Error('boom'); } } as unknown as BrainEngine;
+    const result = await rawProvenanceCheck(broken);
+    expect(result.status).toBe('warn');
+    expect(result.message).toContain('Could not check');
+  });
+
+  test('raw_provenance is categorized as a brain check', () => {
+    expect(categorizeCheck('raw_provenance')).toBe('brain');
+  });
+});

test/extract/receipt-writer.test.ts

@@ -115,6 +115,11 @@ describe('writeReceipt — frontmatter D-EXTRACT-19 belt+suspenders', () => {
     // belt + suspenders: both anti-loop flags are present
     expect(page.frontmatter?.type).toBe('extract_receipt');
     expect(page.frontmatter?.dream_generated).toBe(true);
+    // #1978: receipts are operation records, not derived documents —
+    // explicit raw-trace exemption so the doctor raw_provenance check
+    // (warn-only v1) stays quiet.
+    expect(page.frontmatter?.raw_trace_exempt).toBe(true);
+    expect(typeof page.frontmatter?.raw_trace_exempt_reason).toBe('string');
   });
 
   test('stamps optional model_id + eval_pass + eval_score when supplied', async () => {