fix(cycle): extract_atoms stamps concepts so synthesize_concepts has material (#2123) (#2124) (#3308)

synthesize-concepts.ts's design comment says extract_atoms stamps a
`concepts:` frontmatter field on each atom and :92 consumes ONLY that
field — but the extractor never wrote it, so the atoms → concepts
pipeline was dead end-to-end: every cycle reported "synthesize_concepts:
skipped — no atoms with concept refs" no matter how many atoms
accumulated (696 page-derived atoms / 0 with concepts on our production
brain before an external backfill).

Fix, all on the extractor side (no synthesize change needed):
- EXTRACT_PROMPT asks for `concepts` (1-3 kebab-case TOPIC labels) with
  an explicit reuse-over-coinage instruction — labels must cluster,
  since synthesize_concepts only materializes groups of >=2.
- parseAtomsResponse validates labels (kebab regex, max 3, drop
  invalid; empty -> undefined).
- The putPage frontmatter write stamps `concepts` alongside lesson /
  source_quote.

Tests: 4 parse cases + an end-to-end regression that goes extractor ->
real frontmatter -> synthesize_concepts' OWN DB query path -> concept
page. The existing tests fed synthesize via the `_atoms` seam, which is
exactly how this gap survived.

Validated in production ahead of this PR by stamping the same shape
externally: the next synthesize_concepts run wrote 33 concept pages
(T2=7/T3=26) from 60 stamped atoms, zero failures.

Co-authored-by: 陈源泉 <84364275+ChenyqThu@users.noreply.github.com>
Co-authored-by: 陈源泉 <chenyuanquan@chenyuanquandeMac-mini.local>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

src/core/cycle/extract-atoms.ts

@@ -171,10 +171,20 @@ interface ExtractedAtom {
   body: string;
   source_quote?: string;
   lesson?: string;
+  /**
+   * 1-3 kebab-case topic labels for concept clustering. Consumed by
+   * synthesize_concepts (groups atoms by `frontmatter.concepts`; only
+   * labels shared by >=2 atoms materialize a concept page, so the prompt
+   * biases reuse-over-coinage). #2123.
+   */
+  concepts?: string[];
   virality_score?: number;
   emotional_register?: string;
 }
 
+/** kebab-case validator for concept labels ("captive-portal", "channel-pricing"). */
+const CONCEPT_LABEL_RE = /^[a-z0-9]+(-[a-z0-9]+)*$/;
+
 const EXTRACT_PROMPT = `You extract atomic content nuggets from a transcript.
 
 An atom is a single-source, self-contained idea that could become a tweet,
@@ -185,12 +195,17 @@ quote, or short essay angle. Each atom must:
 
 Output a JSON array of atoms (1-3 per transcript, never more than 3).
 Each atom: {title (≤80 chars), atom_type, body (2-4 sentences),
-source_quote (verbatim ≤200 chars), lesson (one sentence), virality_score
-(0-100), emotional_register (one of: shocking, inspiring, funny, sobering,
-practical, controversial)}.
+source_quote (verbatim ≤200 chars), lesson (one sentence), concepts
+(1-3 topic labels), virality_score (0-100), emotional_register (one of:
+shocking, inspiring, funny, sobering, practical, controversial)}.
 
 atom_type MUST be one of: ${ATOM_TYPES.join(', ')}.
 
+concepts are kebab-case English TOPIC labels used to cluster atoms into
+concept pages (e.g. "captive-portal", "channel-pricing-strategy") — never
+entity or brand names. Use the same label for the same topic across atoms;
+prefer a label you already used over coining a near-synonym.
+
 Output ONLY the JSON array, no prose.`;
 
 interface DiscoveredPage {
@@ -600,6 +615,7 @@ export async function runPhaseExtractAtoms(
                 source_hash: item.contentHash.slice(0, 16),
                 ...(atom.source_quote && { source_quote: atom.source_quote }),
                 ...(atom.lesson && { lesson: atom.lesson }),
+                ...(atom.concepts && atom.concepts.length > 0 && { concepts: atom.concepts }),
                 ...(atom.virality_score !== undefined && { virality_score: atom.virality_score }),
                 ...(atom.emotional_register && { emotional_register: atom.emotional_register }),
                 extracted_at: new Date().toISOString(),
@@ -736,6 +752,13 @@ export function parseAtomsResponse(raw: string): ExtractedAtom[] {
       body,
       source_quote: typeof obj.source_quote === 'string' ? obj.source_quote.slice(0, 500) : undefined,
       lesson: typeof obj.lesson === 'string' ? obj.lesson : undefined,
+      concepts: (() => {
+        if (!Array.isArray(obj.concepts)) return undefined;
+        const labels = obj.concepts
+          .filter((c): c is string => typeof c === 'string' && CONCEPT_LABEL_RE.test(c))
+          .slice(0, 3);
+        return labels.length > 0 ? labels : undefined;
+      })(),
       virality_score:
         typeof obj.virality_score === 'number' &&
         obj.virality_score >= 0 &&

test/cycle/extract-atoms-synthesize-concepts.test.ts

@@ -370,3 +370,69 @@ describe('v0.41 T6: runPhaseSynthesizeConcepts via stubbed chat', () => {
     expect((page[0].fm as Record<string, unknown>).tier).toBe('T1');
   });
 });
+
+// #2123 — extract_atoms must stamp `concepts` so synthesize_concepts has
+// material. The pre-fix pipeline was broken end-to-end: the extractor
+// never wrote the field, and every synthesize_concepts cycle skipped with
+// "no atoms with concept refs". The earlier describe blocks feed
+// synthesize via the `_atoms` seam, which is exactly how the gap survived
+// — so the last test here goes extractor → REAL frontmatter → real DB
+// query path → concept page.
+describe('#2123: concepts label parsing', () => {
+  test('keeps valid kebab-case labels', () => {
+    const raw = `[{"title":"T","atom_type":"insight","body":"b","concepts":["captive-portal","tls-certificates"]}]`;
+    expect(parseAtomsResponse(raw)[0].concepts).toEqual(['captive-portal', 'tls-certificates']);
+  });
+
+  test('filters non-kebab labels, keeps the rest', () => {
+    const raw = `[{"title":"T","atom_type":"insight","body":"b","concepts":["Captive Portal","tp_link","UPPER","valid-label"]}]`;
+    expect(parseAtomsResponse(raw)[0].concepts).toEqual(['valid-label']);
+  });
+
+  test('truncates to 3 labels', () => {
+    const raw = `[{"title":"T","atom_type":"insight","body":"b","concepts":["a","b","c","d","e"]}]`;
+    expect(parseAtomsResponse(raw)[0].concepts).toEqual(['a', 'b', 'c']);
+  });
+
+  test('absent / non-array / all-invalid → undefined', () => {
+    expect(parseAtomsResponse(`[{"title":"T","atom_type":"insight","body":"b"}]`)[0].concepts).toBeUndefined();
+    expect(parseAtomsResponse(`[{"title":"T","atom_type":"insight","body":"b","concepts":"not-an-array"}]`)[0].concepts).toBeUndefined();
+    expect(parseAtomsResponse(`[{"title":"T","atom_type":"insight","body":"b","concepts":["Bad Label!"]}]`)[0].concepts).toBeUndefined();
+  });
+});
+
+describe('#2123: extractor stamps concepts → synthesize_concepts consumes via real DB path', () => {
+  test('end-to-end: atoms with shared label materialize a concept page', async () => {
+    const chat = stubChat(`[
+      {"title":"Cert warning on guest wifi","atom_type":"insight","body":"Portal redirects to an IP-based HTTPS URL.","concepts":["captive-portal"]},
+      {"title":"iPhone portal popup is flaky","atom_type":"critique","body":"CNA probe behavior differs across iOS versions.","concepts":["captive-portal"]}
+    ]`);
+    const extract = await runPhaseExtractAtoms(engine, {
+      _transcripts: [{ filePath: '/fake/notes.txt', content: 'content', contentHash: 'cc2123' }],
+      _pages: [],
+      _chat: chat,
+    });
+    expect(extract.status).toBe('ok');
+    expect(extract.details?.atoms_extracted).toBe(2);
+
+    // Frontmatter really carries the label (a jsonb array, not a string).
+    const stamped = await engine.executeRaw<{ concepts: unknown }>(
+      `SELECT frontmatter->'concepts' AS concepts FROM pages WHERE type = 'atom'`,
+    );
+    expect(stamped.length).toBe(2);
+    for (const row of stamped) {
+      const arr = typeof row.concepts === 'string' ? JSON.parse(row.concepts) : row.concepts;
+      expect(arr).toEqual(['captive-portal']);
+    }
+
+    // NO _atoms seam: synthesize discovers the atoms through its own
+    // DB query — this is the path that was dead before the fix.
+    const synth = await runPhaseSynthesizeConcepts(engine, { _chat: stubChat('unused — T3 is deterministic') });
+    expect(synth.status).toBe('ok');
+    expect(synth.details?.concepts_written).toBe(1);
+    const concept = await engine.executeRaw<{ slug: string }>(
+      `SELECT slug FROM pages WHERE slug = 'concepts/captive-portal' AND type = 'concept'`,
+    );
+    expect(concept.length).toBe(1);
+  });
+});