feat(conversation-facts): parse Slack block format + route granular collector page-types (#2357)

extract-conversation-facts extracted nothing on brains that store chat in the
collector's native page types. Two stacked gaps:

- Type routing: the allowlist exact-matched {conversation,meeting,slack,email}
  against pages.type and passed each straight to listPages({type}), so
  --types slack matched zero rows on a brain carrying slack-dm-day /
  slack-thread / email-digest. Add ALLOWED_TYPE_ALIASES + pageTypesForAllowed()
  to expand logical -> concrete (canonical name first so consolidated brains are
  unaffected), wired into both the single-slug filter and the listPages loop.

- Block-format parsing: the 14 built-in patterns are single-line; the Slack
  collector emits a header + indented-body block (`- **Name** (Mon 11:18)` then
  body on following lines) that none match -> phase:'no_match', 0 messages, and
  the LLM fallback is not wired. Add normalize-block.ts, a strict-no-op pre-pass
  in parseConversation that collapses the block into the canonical
  `**Name** (HH:MM): body` line the bold-paren-time pattern handles; the
  per-message date fills in downstream via fallbackDate. 12h am/pm normalized to
  24h; day-of-week dropped.

Verified on a 13.7K-page comms brain whose facts table was empty: a 12-page
Slack sample went 0/12 parsed (no_match) -> 12/12 (regex_match), 103 messages,
13 segments; extraction wrote 58 facts across 16 entities (~$0.09) and
find_trajectory returns a populated points list for a local/owner caller where
it previously returned empty.

Tests: +13 normalize-block (detection, multi-paragraph collapse, 12h->24h,
no-op on canonical, parseConversation integration) + 7 pageTypesForAllowed.
typecheck clean; verify 30/30.


Claude-Session: https://claude.ai/code/session_01E5wtDU4ZLKewXUYkPLQHSy

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Garry Tan <garrytan@gmail.com>
Co-authored-by: Time Attakc <89218912+time-attack@users.noreply.github.com>

node_modules

@@ -0,0 +1 @@
+/tmp/fleet/repo/node_modules

src/commands/extract-conversation-facts.ts

@@ -149,6 +149,40 @@ export const ALLOWED_TYPES = [
 ] as const;
 export type AllowedType = (typeof ALLOWED_TYPES)[number];
 
+/**
+ * Granular collector page-types that alias into each canonical conversation
+ * bucket. The v2 type-consolidation pack retypes these to the canonical names
+ * (`slack-dm-day`/`slack-thread` → `slack`, `email-digest` → `email`), but a
+ * brain that hasn't run that pack still carries the collector's granular types
+ * in `pages.type`. Without this expansion, `listPages({ type: 'slack' })`
+ * matches zero rows on such brains and the whole comms corpus is silently
+ * skipped (facts stay empty → `find_trajectory` returns nothing). The canonical
+ * name is always included first so consolidated brains keep working unchanged.
+ */
+export const ALLOWED_TYPE_ALIASES: Record<AllowedType, readonly string[]> = {
+  conversation: ['conversation'],
+  meeting: ['meeting'],
+  slack: ['slack', 'slack-dm-day', 'slack-thread'],
+  email: ['email', 'email-digest'],
+  imessage: ['imessage'],
+  'imessage-daily': ['imessage-daily'],
+};
+
+/**
+ * Expand the requested logical types to the concrete `pages.type` values to
+ * enumerate, canonical-first and de-duplicated. Unknown types pass through
+ * unchanged so an explicit override is never dropped.
+ */
+export function pageTypesForAllowed(types: readonly AllowedType[]): string[] {
+  const out: string[] = [];
+  for (const t of types) {
+    for (const concrete of ALLOWED_TYPE_ALIASES[t] ?? [t]) {
+      if (!out.includes(concrete)) out.push(concrete);
+    }
+  }
+  return out;
+}
+
 /**
  * Pagination batch size for listPages enumeration. Per-batch memory
  * worst case = BATCH × MAX_PAGE_BODY_BYTES = 250MB at default 10
@@ -1264,13 +1298,18 @@ export async function runExtractConversationFactsCore(
       }
     };
 
+    // Expand logical types (conversation/meeting/slack/email) to the concrete
+    // `pages.type` values to enumerate, so brains on the granular collector
+    // types are not silently skipped (see ALLOWED_TYPE_ALIASES).
+    const concreteTypes = pageTypesForAllowed(types);
+
     if (opts.slug) {
       const page = await engine.getPage(opts.slug, { sourceId });
       if (!page) {
         result.pages_skipped_disappeared++;
         return;
       }
-      if (!types.includes(page.type as AllowedType)) {
+      if (!concreteTypes.includes(page.type)) {
         result.pages_skipped++;
         return;
       }
@@ -1284,7 +1323,7 @@ export async function runExtractConversationFactsCore(
       // honors AbortSignal at each claim boundary and threads
       // BudgetExhausted abort (D13) automatically.
       let processedPagesCount = 0;
-      pageLoop: for (const type of types) {
+      pageLoop: for (const type of concreteTypes) {
         let offset = 0;
         // eslint-disable-next-line no-constant-condition
         while (true) {

src/core/conversation-parser/normalize-block.ts

@@ -0,0 +1,94 @@
+/**
+ * Block-format conversation normalizer.
+ *
+ * Some chat exports — notably the Slack collector gbrain's own ingestion
+ * uses — emit a HEADER + indented-body BLOCK per message instead of the
+ * single-line `**Name** (time): body` shape the built-in patterns
+ * (`builtins.ts`) recognize:
+ *
+ *     - **Theo** (Mon 11:18)
+ *       Hey everyone — quick update on the renewal.
+ *
+ *       Second paragraph of the same message.
+ *     - **Juan** (Mon 11:20)
+ *       Reply body...
+ *
+ * None of the 14 line-oriented built-ins match this: a leading `- ` list
+ * marker, a day-of-week + time with no trailing colon, and the message body on
+ * the following indented lines. Result: `phase: 'no_match'`, zero messages,
+ * and the whole comms corpus is silently un-extractable (facts stay empty →
+ * `find_trajectory` returns nothing).
+ *
+ * This collapses each block into the canonical `**Name** (HH:MM): <body joined
+ * to one line>` shape so the existing `bold-paren-time` pattern matches; the
+ * per-message date fills in downstream via `fallbackDate` (the page date).
+ *
+ * STRICT no-op unless the block signature is present: the header regex requires
+ * the paren-group to END the line (no inline `: body`), which is exactly what
+ * the single-line patterns always produce — so feeding already-canonical
+ * content through this function returns it unchanged.
+ */
+
+// `- **Name** (Mon 11:18)` / `- **Name** (11:18 AM)` / `- **Name** (16:36)`.
+// Day-of-week optional; 12h/24h time; optional am/pm; the line ENDS at the
+// close paren (no inline `: body` — that is what distinguishes a block header
+// from the single-line `**Name** (time): body` patterns).
+const BLOCK_HEADER =
+  /^\s*-\s+\*\*(.+?)\*\*\s+\((?:[A-Za-z]{2,9}\.?\s+)?(\d{1,2}):(\d{2})(?::\d{2})?\s*([AaPp][Mm])?\)\s*$/;
+
+/** True when at least one line is a block-format message header. */
+export function looksLikeBlockConversation(body: string): boolean {
+  for (const line of body.split('\n')) {
+    if (BLOCK_HEADER.test(line)) return true;
+  }
+  return false;
+}
+
+function to24h(hour: number, ampm?: string): number {
+  if (!ampm) return hour;
+  const lower = ampm.toLowerCase();
+  if (lower === 'pm' && hour < 12) return hour + 12;
+  if (lower === 'am' && hour === 12) return 0;
+  return hour;
+}
+
+/**
+ * Collapse block-format messages into canonical single-line `**Name** (HH:MM):
+ * body` lines. Returns `body` unchanged when no block header is present.
+ */
+export function normalizeBlockConversation(body: string): string {
+  if (!looksLikeBlockConversation(body)) return body;
+
+  const lines = body.split('\n');
+  const out: string[] = [];
+  let current: { name: string; time: string } | null = null;
+  let bodyParts: string[] = [];
+
+  const flush = () => {
+    if (current) {
+      const text = bodyParts.join(' ').replace(/\s+/g, ' ').trim();
+      out.push(`**${current.name}** (${current.time}): ${text}`);
+    }
+    current = null;
+    bodyParts = [];
+  };
+
+  for (const line of lines) {
+    const m = BLOCK_HEADER.exec(line);
+    if (m) {
+      flush();
+      const hour = to24h(parseInt(m[2], 10), m[4]);
+      const time = `${String(hour).padStart(2, '0')}:${m[3]}`;
+      current = { name: m[1].trim(), time };
+    } else if (current) {
+      // Body line of the current message. Drop blank lines; keep the rest.
+      const trimmed = line.trim();
+      if (trimmed) bodyParts.push(trimmed);
+    }
+    // Lines before the first header (page title, leading blanks) are dropped —
+    // they never matched a pattern anyway.
+  }
+  flush();
+
+  return out.length > 0 ? out.join('\n') : body;
+}

src/core/conversation-parser/parse.ts

@@ -29,6 +29,7 @@ import {
   BUILTIN_PATTERNS,
   cleanSpeaker,
 } from './builtins.ts';
+import { normalizeBlockConversation } from './normalize-block.ts';
 import type {
   DateContext,
   MatchedMessage,
@@ -473,6 +474,12 @@ export function parseConversation(
     return { messages: [], phase: 'no_match' };
   }
 
+  // Pre-pass: collapse block-format chat exports (header + indented body, e.g.
+  // the Slack collector's `- **Name** (Mon 11:18)\n  body…`) into the canonical
+  // single-line shape the built-in patterns recognize. Strict no-op when no
+  // block header is present, so already-canonical content is untouched.
+  body = normalizeBlockConversation(body);
+
   const dateCtx = deriveDateContext(opts);
 
   // Assemble candidate pool: built-ins (minus disabled) + user patterns.

test/conversation-parser-normalize-block.test.ts

@@ -0,0 +1,81 @@
+import { describe, test, expect } from 'bun:test';
+import {
+  normalizeBlockConversation,
+  looksLikeBlockConversation,
+} from '../src/core/conversation-parser/normalize-block.ts';
+import { parseConversation } from '../src/core/conversation-parser/parse.ts';
+
+// A realistic Slack-collector page body (the format gbrain's own collector emits).
+const SLACK_DM = `# DM (group) with Hugh, Karyshma, Theo — 2026-06-15
+
+- **Theo** (Mon 11:18)
+  Hey everyone — quick note on the *real fiscal value* we surface after accounting.
+
+  It's a huge win at renewal and dents churn.
+- **Juan** (Mon 11:20)
+  Agreed. Let's make sure we capture it for all ongoing customers.`;
+
+describe('looksLikeBlockConversation', () => {
+  test('detects the block header signature', () => {
+    expect(looksLikeBlockConversation(SLACK_DM)).toBe(true);
+    expect(looksLikeBlockConversation('- **Theo** (16:36)\n  body')).toBe(true);
+    expect(looksLikeBlockConversation('- **Theo** (11:18 AM)\n  body')).toBe(true);
+  });
+
+  test('is false for canonical single-line content (no false trigger)', () => {
+    expect(looksLikeBlockConversation('**Theo** (11:18): hi there')).toBe(false);
+    expect(looksLikeBlockConversation('**Theo** (2026-06-15 11:18): hi')).toBe(false);
+    expect(looksLikeBlockConversation('just some prose with no chat at all')).toBe(false);
+  });
+});
+
+describe('normalizeBlockConversation', () => {
+  test('collapses header + indented multi-paragraph body to one canonical line', () => {
+    const out = normalizeBlockConversation(SLACK_DM).split('\n');
+    expect(out).toEqual([
+      "**Theo** (11:18): Hey everyone — quick note on the *real fiscal value* we surface after accounting. It's a huge win at renewal and dents churn.",
+      "**Juan** (11:20): Agreed. Let's make sure we capture it for all ongoing customers.",
+    ]);
+  });
+
+  test('drops the page-title line and leading blanks', () => {
+    const out = normalizeBlockConversation(SLACK_DM);
+    expect(out.startsWith('# DM')).toBe(false);
+    expect(out.startsWith('**Theo**')).toBe(true);
+  });
+
+  test('converts 12h am/pm to 24h', () => {
+    expect(normalizeBlockConversation('- **A** (1:05 PM)\n  hi')).toBe('**A** (13:05): hi');
+    expect(normalizeBlockConversation('- **A** (12:00 AM)\n  midnight')).toBe('**A** (00:00): midnight');
+    expect(normalizeBlockConversation('- **A** (12:30 PM)\n  noon-ish')).toBe('**A** (12:30): noon-ish');
+  });
+
+  test('keeps day-of-week out of the emitted time', () => {
+    expect(normalizeBlockConversation('- **A** (Tue 09:07)\n  morning')).toBe('**A** (09:07): morning');
+  });
+
+  test('is a strict no-op on canonical single-line content', () => {
+    const canonical = '**Theo** (11:18): hi\n**Juan** (11:20): yo';
+    expect(normalizeBlockConversation(canonical)).toBe(canonical);
+  });
+
+  test('a message with no body emits an empty-body line', () => {
+    expect(normalizeBlockConversation('- **A** (10:00)')).toBe('**A** (10:00): ');
+  });
+});
+
+describe('parseConversation integration — block format now yields messages', () => {
+  test('Slack-collector body parses to 2 messages via the normalize pre-pass', () => {
+    const res = parseConversation(SLACK_DM, { fallbackDate: '2026-06-15' });
+    expect(res.messages.length).toBe(2);
+    expect(res.messages[0].speaker).toBe('Theo');
+    expect(res.messages[1].speaker).toBe('Juan');
+    expect(res.phase).not.toBe('no_match');
+  });
+
+  test('canonical content still parses unchanged (no regression)', () => {
+    const res = parseConversation('**Theo** (11:18): hi there', { fallbackDate: '2026-06-15' });
+    expect(res.messages.length).toBe(1);
+    expect(res.messages[0].speaker).toBe('Theo');
+  });
+});

test/extract-conversation-facts.test.ts

@@ -39,10 +39,53 @@ import {
   NON_EXTRACTABLE_AUDIT_SOURCE,
   PER_SEGMENT_SOURCE_PREFIX,
   ALLOWED_TYPES,
+  pageTypesForAllowed,
+  ALLOWED_TYPE_ALIASES,
 } from '../src/commands/extract-conversation-facts.ts';
 import { _resetLlmCacheForTests } from '../src/core/conversation-parser/llm-base.ts';
 import { BudgetExhausted } from '../src/core/budget/budget-tracker.ts';
 
+// ---------------------------------------------------------------------------
+// pageTypesForAllowed — logical→concrete page-type expansion.
+// ---------------------------------------------------------------------------
+
+describe('pageTypesForAllowed', () => {
+  test('expands slack to canonical + granular collector types', () => {
+    expect(pageTypesForAllowed(['slack'])).toEqual(['slack', 'slack-dm-day', 'slack-thread']);
+  });
+
+  test('expands email to canonical + granular collector types', () => {
+    expect(pageTypesForAllowed(['email'])).toEqual(['email', 'email-digest']);
+  });
+
+  test('canonical-only types pass through unchanged', () => {
+    expect(pageTypesForAllowed(['meeting'])).toEqual(['meeting']);
+    expect(pageTypesForAllowed(['conversation'])).toEqual(['conversation']);
+  });
+
+  test('canonical name is always first so consolidated brains keep working', () => {
+    expect(pageTypesForAllowed(['slack'])[0]).toBe('slack');
+    expect(pageTypesForAllowed(['email'])[0]).toBe('email');
+  });
+
+  test('multiple logical types flatten and de-duplicate', () => {
+    const got = pageTypesForAllowed(['slack', 'email', 'meeting']);
+    expect(got).toEqual(['slack', 'slack-dm-day', 'slack-thread', 'email', 'email-digest', 'meeting']);
+    // no duplicates
+    expect(new Set(got).size).toBe(got.length);
+  });
+
+  test('every ALLOWED_TYPE_ALIASES entry lists its canonical name first', () => {
+    for (const [canonical, concretes] of Object.entries(ALLOWED_TYPE_ALIASES)) {
+      expect(concretes[0]).toBe(canonical);
+    }
+  });
+
+  test('empty input yields empty output', () => {
+    expect(pageTypesForAllowed([])).toEqual([]);
+  });
+});
+
 // ---------------------------------------------------------------------------
 // Fixture helpers.
 // ---------------------------------------------------------------------------