From faf5cdba54bcba6d7be2a5e94230aa9bd7223377 Mon Sep 17 00:00:00 2001 From: johnnymn3monic Date: Tue, 28 Jul 2026 02:06:01 +0100 Subject: [PATCH] feat(conversation-facts): parse Slack block format + route granular collector page-types (#2357) extract-conversation-facts extracted nothing on brains that store chat in the collector's native page types. Two stacked gaps: - Type routing: the allowlist exact-matched {conversation,meeting,slack,email} against pages.type and passed each straight to listPages({type}), so --types slack matched zero rows on a brain carrying slack-dm-day / slack-thread / email-digest. Add ALLOWED_TYPE_ALIASES + pageTypesForAllowed() to expand logical -> concrete (canonical name first so consolidated brains are unaffected), wired into both the single-slug filter and the listPages loop. - Block-format parsing: the 14 built-in patterns are single-line; the Slack collector emits a header + indented-body block (`- **Name** (Mon 11:18)` then body on following lines) that none match -> phase:'no_match', 0 messages, and the LLM fallback is not wired. Add normalize-block.ts, a strict-no-op pre-pass in parseConversation that collapses the block into the canonical `**Name** (HH:MM): body` line the bold-paren-time pattern handles; the per-message date fills in downstream via fallbackDate. 12h am/pm normalized to 24h; day-of-week dropped. Verified on a 13.7K-page comms brain whose facts table was empty: a 12-page Slack sample went 0/12 parsed (no_match) -> 12/12 (regex_match), 103 messages, 13 segments; extraction wrote 58 facts across 16 entities (~$0.09) and find_trajectory returns a populated points list for a local/owner caller where it previously returned empty. Tests: +13 normalize-block (detection, multi-paragraph collapse, 12h->24h, no-op on canonical, parseConversation integration) + 7 pageTypesForAllowed. typecheck clean; verify 30/30. Claude-Session: https://claude.ai/code/session_01E5wtDU4ZLKewXUYkPLQHSy Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: Garry Tan Co-authored-by: Time Attakc <89218912+time-attack@users.noreply.github.com> --- node_modules | 1 + src/commands/extract-conversation-facts.ts | 43 ++++++++- .../conversation-parser/normalize-block.ts | 94 +++++++++++++++++++ src/core/conversation-parser/parse.ts | 7 ++ ...onversation-parser-normalize-block.test.ts | 81 ++++++++++++++++ test/extract-conversation-facts.test.ts | 43 +++++++++ 6 files changed, 267 insertions(+), 2 deletions(-) create mode 120000 node_modules create mode 100644 src/core/conversation-parser/normalize-block.ts create mode 100644 test/conversation-parser-normalize-block.test.ts diff --git a/node_modules b/node_modules new file mode 120000 index 000000000..96daf4519 --- /dev/null +++ b/node_modules @@ -0,0 +1 @@ +/tmp/fleet/repo/node_modules \ No newline at end of file diff --git a/src/commands/extract-conversation-facts.ts b/src/commands/extract-conversation-facts.ts index 9ee54636d..51bd6b201 100644 --- a/src/commands/extract-conversation-facts.ts +++ b/src/commands/extract-conversation-facts.ts @@ -149,6 +149,40 @@ export const ALLOWED_TYPES = [ ] as const; export type AllowedType = (typeof ALLOWED_TYPES)[number]; +/** + * Granular collector page-types that alias into each canonical conversation + * bucket. The v2 type-consolidation pack retypes these to the canonical names + * (`slack-dm-day`/`slack-thread` → `slack`, `email-digest` → `email`), but a + * brain that hasn't run that pack still carries the collector's granular types + * in `pages.type`. Without this expansion, `listPages({ type: 'slack' })` + * matches zero rows on such brains and the whole comms corpus is silently + * skipped (facts stay empty → `find_trajectory` returns nothing). The canonical + * name is always included first so consolidated brains keep working unchanged. + */ +export const ALLOWED_TYPE_ALIASES: Record = { + conversation: ['conversation'], + meeting: ['meeting'], + slack: ['slack', 'slack-dm-day', 'slack-thread'], + email: ['email', 'email-digest'], + imessage: ['imessage'], + 'imessage-daily': ['imessage-daily'], +}; + +/** + * Expand the requested logical types to the concrete `pages.type` values to + * enumerate, canonical-first and de-duplicated. Unknown types pass through + * unchanged so an explicit override is never dropped. + */ +export function pageTypesForAllowed(types: readonly AllowedType[]): string[] { + const out: string[] = []; + for (const t of types) { + for (const concrete of ALLOWED_TYPE_ALIASES[t] ?? [t]) { + if (!out.includes(concrete)) out.push(concrete); + } + } + return out; +} + /** * Pagination batch size for listPages enumeration. Per-batch memory * worst case = BATCH × MAX_PAGE_BODY_BYTES = 250MB at default 10 @@ -1264,13 +1298,18 @@ export async function runExtractConversationFactsCore( } }; + // Expand logical types (conversation/meeting/slack/email) to the concrete + // `pages.type` values to enumerate, so brains on the granular collector + // types are not silently skipped (see ALLOWED_TYPE_ALIASES). + const concreteTypes = pageTypesForAllowed(types); + if (opts.slug) { const page = await engine.getPage(opts.slug, { sourceId }); if (!page) { result.pages_skipped_disappeared++; return; } - if (!types.includes(page.type as AllowedType)) { + if (!concreteTypes.includes(page.type)) { result.pages_skipped++; return; } @@ -1284,7 +1323,7 @@ export async function runExtractConversationFactsCore( // honors AbortSignal at each claim boundary and threads // BudgetExhausted abort (D13) automatically. let processedPagesCount = 0; - pageLoop: for (const type of types) { + pageLoop: for (const type of concreteTypes) { let offset = 0; // eslint-disable-next-line no-constant-condition while (true) { diff --git a/src/core/conversation-parser/normalize-block.ts b/src/core/conversation-parser/normalize-block.ts new file mode 100644 index 000000000..b5165cd0f --- /dev/null +++ b/src/core/conversation-parser/normalize-block.ts @@ -0,0 +1,94 @@ +/** + * Block-format conversation normalizer. + * + * Some chat exports — notably the Slack collector gbrain's own ingestion + * uses — emit a HEADER + indented-body BLOCK per message instead of the + * single-line `**Name** (time): body` shape the built-in patterns + * (`builtins.ts`) recognize: + * + * - **Theo** (Mon 11:18) + * Hey everyone — quick update on the renewal. + * + * Second paragraph of the same message. + * - **Juan** (Mon 11:20) + * Reply body... + * + * None of the 14 line-oriented built-ins match this: a leading `- ` list + * marker, a day-of-week + time with no trailing colon, and the message body on + * the following indented lines. Result: `phase: 'no_match'`, zero messages, + * and the whole comms corpus is silently un-extractable (facts stay empty → + * `find_trajectory` returns nothing). + * + * This collapses each block into the canonical `**Name** (HH:MM): ` shape so the existing `bold-paren-time` pattern matches; the + * per-message date fills in downstream via `fallbackDate` (the page date). + * + * STRICT no-op unless the block signature is present: the header regex requires + * the paren-group to END the line (no inline `: body`), which is exactly what + * the single-line patterns always produce — so feeding already-canonical + * content through this function returns it unchanged. + */ + +// `- **Name** (Mon 11:18)` / `- **Name** (11:18 AM)` / `- **Name** (16:36)`. +// Day-of-week optional; 12h/24h time; optional am/pm; the line ENDS at the +// close paren (no inline `: body` — that is what distinguishes a block header +// from the single-line `**Name** (time): body` patterns). +const BLOCK_HEADER = + /^\s*-\s+\*\*(.+?)\*\*\s+\((?:[A-Za-z]{2,9}\.?\s+)?(\d{1,2}):(\d{2})(?::\d{2})?\s*([AaPp][Mm])?\)\s*$/; + +/** True when at least one line is a block-format message header. */ +export function looksLikeBlockConversation(body: string): boolean { + for (const line of body.split('\n')) { + if (BLOCK_HEADER.test(line)) return true; + } + return false; +} + +function to24h(hour: number, ampm?: string): number { + if (!ampm) return hour; + const lower = ampm.toLowerCase(); + if (lower === 'pm' && hour < 12) return hour + 12; + if (lower === 'am' && hour === 12) return 0; + return hour; +} + +/** + * Collapse block-format messages into canonical single-line `**Name** (HH:MM): + * body` lines. Returns `body` unchanged when no block header is present. + */ +export function normalizeBlockConversation(body: string): string { + if (!looksLikeBlockConversation(body)) return body; + + const lines = body.split('\n'); + const out: string[] = []; + let current: { name: string; time: string } | null = null; + let bodyParts: string[] = []; + + const flush = () => { + if (current) { + const text = bodyParts.join(' ').replace(/\s+/g, ' ').trim(); + out.push(`**${current.name}** (${current.time}): ${text}`); + } + current = null; + bodyParts = []; + }; + + for (const line of lines) { + const m = BLOCK_HEADER.exec(line); + if (m) { + flush(); + const hour = to24h(parseInt(m[2], 10), m[4]); + const time = `${String(hour).padStart(2, '0')}:${m[3]}`; + current = { name: m[1].trim(), time }; + } else if (current) { + // Body line of the current message. Drop blank lines; keep the rest. + const trimmed = line.trim(); + if (trimmed) bodyParts.push(trimmed); + } + // Lines before the first header (page title, leading blanks) are dropped — + // they never matched a pattern anyway. + } + flush(); + + return out.length > 0 ? out.join('\n') : body; +} diff --git a/src/core/conversation-parser/parse.ts b/src/core/conversation-parser/parse.ts index 66aa76745..c12b38ba4 100644 --- a/src/core/conversation-parser/parse.ts +++ b/src/core/conversation-parser/parse.ts @@ -29,6 +29,7 @@ import { BUILTIN_PATTERNS, cleanSpeaker, } from './builtins.ts'; +import { normalizeBlockConversation } from './normalize-block.ts'; import type { DateContext, MatchedMessage, @@ -473,6 +474,12 @@ export function parseConversation( return { messages: [], phase: 'no_match' }; } + // Pre-pass: collapse block-format chat exports (header + indented body, e.g. + // the Slack collector's `- **Name** (Mon 11:18)\n body…`) into the canonical + // single-line shape the built-in patterns recognize. Strict no-op when no + // block header is present, so already-canonical content is untouched. + body = normalizeBlockConversation(body); + const dateCtx = deriveDateContext(opts); // Assemble candidate pool: built-ins (minus disabled) + user patterns. diff --git a/test/conversation-parser-normalize-block.test.ts b/test/conversation-parser-normalize-block.test.ts new file mode 100644 index 000000000..1fbc90950 --- /dev/null +++ b/test/conversation-parser-normalize-block.test.ts @@ -0,0 +1,81 @@ +import { describe, test, expect } from 'bun:test'; +import { + normalizeBlockConversation, + looksLikeBlockConversation, +} from '../src/core/conversation-parser/normalize-block.ts'; +import { parseConversation } from '../src/core/conversation-parser/parse.ts'; + +// A realistic Slack-collector page body (the format gbrain's own collector emits). +const SLACK_DM = `# DM (group) with Hugh, Karyshma, Theo — 2026-06-15 + +- **Theo** (Mon 11:18) + Hey everyone — quick note on the *real fiscal value* we surface after accounting. + + It's a huge win at renewal and dents churn. +- **Juan** (Mon 11:20) + Agreed. Let's make sure we capture it for all ongoing customers.`; + +describe('looksLikeBlockConversation', () => { + test('detects the block header signature', () => { + expect(looksLikeBlockConversation(SLACK_DM)).toBe(true); + expect(looksLikeBlockConversation('- **Theo** (16:36)\n body')).toBe(true); + expect(looksLikeBlockConversation('- **Theo** (11:18 AM)\n body')).toBe(true); + }); + + test('is false for canonical single-line content (no false trigger)', () => { + expect(looksLikeBlockConversation('**Theo** (11:18): hi there')).toBe(false); + expect(looksLikeBlockConversation('**Theo** (2026-06-15 11:18): hi')).toBe(false); + expect(looksLikeBlockConversation('just some prose with no chat at all')).toBe(false); + }); +}); + +describe('normalizeBlockConversation', () => { + test('collapses header + indented multi-paragraph body to one canonical line', () => { + const out = normalizeBlockConversation(SLACK_DM).split('\n'); + expect(out).toEqual([ + "**Theo** (11:18): Hey everyone — quick note on the *real fiscal value* we surface after accounting. It's a huge win at renewal and dents churn.", + "**Juan** (11:20): Agreed. Let's make sure we capture it for all ongoing customers.", + ]); + }); + + test('drops the page-title line and leading blanks', () => { + const out = normalizeBlockConversation(SLACK_DM); + expect(out.startsWith('# DM')).toBe(false); + expect(out.startsWith('**Theo**')).toBe(true); + }); + + test('converts 12h am/pm to 24h', () => { + expect(normalizeBlockConversation('- **A** (1:05 PM)\n hi')).toBe('**A** (13:05): hi'); + expect(normalizeBlockConversation('- **A** (12:00 AM)\n midnight')).toBe('**A** (00:00): midnight'); + expect(normalizeBlockConversation('- **A** (12:30 PM)\n noon-ish')).toBe('**A** (12:30): noon-ish'); + }); + + test('keeps day-of-week out of the emitted time', () => { + expect(normalizeBlockConversation('- **A** (Tue 09:07)\n morning')).toBe('**A** (09:07): morning'); + }); + + test('is a strict no-op on canonical single-line content', () => { + const canonical = '**Theo** (11:18): hi\n**Juan** (11:20): yo'; + expect(normalizeBlockConversation(canonical)).toBe(canonical); + }); + + test('a message with no body emits an empty-body line', () => { + expect(normalizeBlockConversation('- **A** (10:00)')).toBe('**A** (10:00): '); + }); +}); + +describe('parseConversation integration — block format now yields messages', () => { + test('Slack-collector body parses to 2 messages via the normalize pre-pass', () => { + const res = parseConversation(SLACK_DM, { fallbackDate: '2026-06-15' }); + expect(res.messages.length).toBe(2); + expect(res.messages[0].speaker).toBe('Theo'); + expect(res.messages[1].speaker).toBe('Juan'); + expect(res.phase).not.toBe('no_match'); + }); + + test('canonical content still parses unchanged (no regression)', () => { + const res = parseConversation('**Theo** (11:18): hi there', { fallbackDate: '2026-06-15' }); + expect(res.messages.length).toBe(1); + expect(res.messages[0].speaker).toBe('Theo'); + }); +}); diff --git a/test/extract-conversation-facts.test.ts b/test/extract-conversation-facts.test.ts index 63057aa78..2434ba0b3 100644 --- a/test/extract-conversation-facts.test.ts +++ b/test/extract-conversation-facts.test.ts @@ -39,10 +39,53 @@ import { NON_EXTRACTABLE_AUDIT_SOURCE, PER_SEGMENT_SOURCE_PREFIX, ALLOWED_TYPES, + pageTypesForAllowed, + ALLOWED_TYPE_ALIASES, } from '../src/commands/extract-conversation-facts.ts'; import { _resetLlmCacheForTests } from '../src/core/conversation-parser/llm-base.ts'; import { BudgetExhausted } from '../src/core/budget/budget-tracker.ts'; +// --------------------------------------------------------------------------- +// pageTypesForAllowed — logical→concrete page-type expansion. +// --------------------------------------------------------------------------- + +describe('pageTypesForAllowed', () => { + test('expands slack to canonical + granular collector types', () => { + expect(pageTypesForAllowed(['slack'])).toEqual(['slack', 'slack-dm-day', 'slack-thread']); + }); + + test('expands email to canonical + granular collector types', () => { + expect(pageTypesForAllowed(['email'])).toEqual(['email', 'email-digest']); + }); + + test('canonical-only types pass through unchanged', () => { + expect(pageTypesForAllowed(['meeting'])).toEqual(['meeting']); + expect(pageTypesForAllowed(['conversation'])).toEqual(['conversation']); + }); + + test('canonical name is always first so consolidated brains keep working', () => { + expect(pageTypesForAllowed(['slack'])[0]).toBe('slack'); + expect(pageTypesForAllowed(['email'])[0]).toBe('email'); + }); + + test('multiple logical types flatten and de-duplicate', () => { + const got = pageTypesForAllowed(['slack', 'email', 'meeting']); + expect(got).toEqual(['slack', 'slack-dm-day', 'slack-thread', 'email', 'email-digest', 'meeting']); + // no duplicates + expect(new Set(got).size).toBe(got.length); + }); + + test('every ALLOWED_TYPE_ALIASES entry lists its canonical name first', () => { + for (const [canonical, concretes] of Object.entries(ALLOWED_TYPE_ALIASES)) { + expect(concretes[0]).toBe(canonical); + } + }); + + test('empty input yields empty output', () => { + expect(pageTypesForAllowed([])).toEqual([]); + }); +}); + // --------------------------------------------------------------------------- // Fixture helpers. // ---------------------------------------------------------------------------