From e5380514011c6ceb824e53127c1025ef57336966 Mon Sep 17 00:00:00 2001 From: pabloglzg Date: Thu, 16 Jul 2026 20:53:40 -0600 Subject: [PATCH] feat(extract): recognize inline [Source: ..., YYYY-MM-DD] citations as timeline entries (#2524) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(extract): recognize inline [Source: ..., YYYY-MM-DD] citations as timeline entries gbrain's own quality conventions (skills/conventions/quality.md) require a dated [Source: ..., YYYY-MM-DD] citation on every brain write, so curated pages are full of dated evidence — but extractTimelineFromContent only recognized the timeline-bullet and date-header formats. A page whose dates all live in citations scored zero timeline coverage in brain_score, and doctor pointed users at a formatting convention their own citations already satisfied in spirit. Format 3 files one entry per citation: date and source from the marker, summary from the annotated line with citation markers stripped. Lines already captured by Format 1 are skipped so a timeline bullet carrying its own citation is not double-filed. Bare citations with no surrounding text are ignored. Idempotency is unchanged: persistence already dedupes at the DB layer. * fix(extract): Format 3 citations also in parseTimelineEntries (db-source + ingest path) The first commit only taught extractTimelineFromContent (fs-source) the citation format; the db-source extract and the ingest path parse through parseTimelineEntries in core/link-extraction.ts, which still could not see citations. Same rules as the fs parser: bullet-captured lines skipped, bare citations ignored, invalid calendar dates rejected; the citation source is preserved in the entry detail. --------- Co-authored-by: pabloglzg <186649799+pabloglzg@users.noreply.github.com> --- src/commands/extract.ts | 32 +++++++++++++++++++++++ src/core/link-extraction.ts | 25 ++++++++++++++++++ test/extract.test.ts | 50 ++++++++++++++++++++++++++++++++++++ test/link-extraction.test.ts | 26 +++++++++++++++++++ 4 files changed, 133 insertions(+) diff --git a/src/commands/extract.ts b/src/commands/extract.ts index 01f5dbe43..6c9a92e6b 100644 --- a/src/commands/extract.ts +++ b/src/commands/extract.ts @@ -494,6 +494,38 @@ export function extractTimelineFromContent(content: string, slug: string): Extra entries.push({ slug, date: match[1], source: 'markdown', summary: match[2].trim(), detail: detail || undefined }); } + // Format 3: Inline citation — [Source: , YYYY-MM-DD] + // + // This is the citation convention gbrain's own quality rules require on + // every brain write (skills/conventions/quality.md), so dated evidence is + // pervasive in curated pages — but until now the extractor could not see + // it, and a page whose dates all live in citations scored zero timeline + // coverage. The entry's summary is the sentence the citation annotates + // (the surrounding line with citation markers stripped). + // + // Lines already captured by Format 1 are skipped: a timeline bullet often + // carries its own [Source: ...] citation, and re-extracting it would file + // a duplicate entry under a different (source, summary) shape that the + // DB-level uniqueness cannot collapse. + const citationPattern = /\[Source:\s*([^\]]+?),\s*(\d{4}-\d{2}-\d{2})\s*\]/g; + const bulletLinePattern = /^-\s+\*\*\d{4}-\d{2}-\d{2}\*\*\s*\|/; + for (const line of content.split(/\r?\n/)) { + if (bulletLinePattern.test(line)) continue; + const lineMatches = [...line.matchAll(citationPattern)]; + if (lineMatches.length === 0) continue; + // Strip every citation marker from the line to leave the annotated text. + const summary = line + .replace(/\[Source:[^\]]*\]/g, '') + .replace(/^[-*>#\s]+/, '') + .replace(/\s+/g, ' ') + .trim() + .slice(0, 300); + if (!summary) continue; // a bare citation with no surrounding text is not an event + for (const m of lineMatches) { + entries.push({ slug, date: m[2], source: m[1].trim().slice(0, 200), summary }); + } + } + return entries; } diff --git a/src/core/link-extraction.ts b/src/core/link-extraction.ts index 8ebbac318..c0fc2644a 100644 --- a/src/core/link-extraction.ts +++ b/src/core/link-extraction.ts @@ -1156,6 +1156,31 @@ export function parseTimelineEntries(content: string): TimelineCandidate[] { result.push({ date, summary, detail: detailLines.join(' ').trim() }); i = j; } + + // Format 3: inline citation — [Source: , YYYY-MM-DD]. The citation + // convention gbrain's own quality rules require on every brain write; + // until now this parser (the db-source extract + ingest path) could not + // see it, so a page whose dates all live in citations scored zero + // timeline coverage. Kept in sync with extractTimelineFromContent's + // Format 3 (the fs-source path). Lines already captured by the timeline + // bullet pass are skipped (a bullet often carries its own citation). + const citationRe = /\[Source:\s*([^\]]+?),\s*(\d{4}-\d{2}-\d{2})\s*\]/g; + for (const line of lines) { + if (TIMELINE_LINE_RE.test(line)) continue; + const matches = [...line.matchAll(citationRe)]; + if (matches.length === 0) continue; + const summary = line + .replace(/\[Source:[^\]]*\]/g, '') + .replace(/^[-*>#\s]+/, '') + .replace(/\s+/g, ' ') + .trim() + .slice(0, 300); + if (!summary) continue; + for (const m of matches) { + if (!isValidDate(m[2])) continue; + result.push({ date: m[2], summary, detail: `Source: ${m[1].trim().slice(0, 200)}` }); + } + } return result; } diff --git a/test/extract.test.ts b/test/extract.test.ts index 1c9b15f14..5764de52b 100644 --- a/test/extract.test.ts +++ b/test/extract.test.ts @@ -135,6 +135,56 @@ describe('extractTimelineFromContent', () => { const entries = extractTimelineFromContent(content, 'test'); expect(entries).toHaveLength(1); }); + + it('extracts inline citation format entries', () => { + const content = `Closed the seed round with fund-a leading. [Source: board meeting notes, 2025-04-02]`; + const entries = extractTimelineFromContent(content, 'deals/acme-seed'); + expect(entries).toHaveLength(1); + expect(entries[0].date).toBe('2025-04-02'); + expect(entries[0].source).toBe('board meeting notes'); + expect(entries[0].summary).toBe('Closed the seed round with fund-a leading.'); + }); + + it('keeps commas inside the citation source', () => { + const content = `Alice joined as CTO. [Source: email from alice-example re: offer, signed, 2025-05-10]`; + const entries = extractTimelineFromContent(content, 'people/alice-example'); + expect(entries).toHaveLength(1); + expect(entries[0].date).toBe('2025-05-10'); + expect(entries[0].source).toBe('email from alice-example re: offer, signed'); + }); + + it('extracts one entry per citation when a line carries several', () => { + const content = `Both sides confirmed the partnership. [Source: call with widget-co, 2025-06-01] [Source: follow-up email, 2025-06-03]`; + const entries = extractTimelineFromContent(content, 'companies/widget-co'); + expect(entries).toHaveLength(2); + expect(entries[0].date).toBe('2025-06-01'); + expect(entries[1].date).toBe('2025-06-03'); + expect(entries[0].summary).toBe(entries[1].summary); + }); + + it('does not double-extract a timeline bullet that carries its own citation', () => { + const content = `- **2025-03-18** | Meeting — Discussed partnership [Source: meeting notes, 2025-03-18]`; + const entries = extractTimelineFromContent(content, 'test'); + expect(entries).toHaveLength(1); // Format 1 only + expect(entries[0].source).toBe('Meeting'); + }); + + it('skips a bare citation with no surrounding text', () => { + const content = `[Source: import batch, 2025-07-01]`; + expect(extractTimelineFromContent(content, 'test')).toHaveLength(0); + }); + + it('ignores citations without a date', () => { + const content = `Some claim here. [Source: undated memo]`; + expect(extractTimelineFromContent(content, 'test')).toHaveLength(0); + }); + + it('strips list markers from the citation summary', () => { + const content = `- Landed the enterprise pilot with acme-example. [Source: CRM update, 2025-08-15]`; + const entries = extractTimelineFromContent(content, 'companies/acme-example'); + expect(entries).toHaveLength(1); + expect(entries[0].summary).toBe('Landed the enterprise pilot with acme-example.'); + }); }); describe('walkMarkdownFiles', () => { diff --git a/test/link-extraction.test.ts b/test/link-extraction.test.ts index 0c180f232..e0a741220 100644 --- a/test/link-extraction.test.ts +++ b/test/link-extraction.test.ts @@ -1265,3 +1265,29 @@ describe("v0.18.0 migration v22 — links_resolution_type", () => { }); }); + +describe('parseTimelineEntries — Format 3: inline [Source: ..., YYYY-MM-DD] citations', () => { + test('extracts an entry from a dated citation', () => { + const entries = parseTimelineEntries('Closed the seed round. [Source: board notes, 2025-04-02]'); + expect(entries).toHaveLength(1); + expect(entries[0].date).toBe('2025-04-02'); + expect(entries[0].summary).toBe('Closed the seed round.'); + expect(entries[0].detail).toBe('Source: board notes'); + }); + + test('keeps commas inside the citation source', () => { + const entries = parseTimelineEntries('Alice joined. [Source: email re: offer, signed, 2025-05-10]'); + expect(entries).toHaveLength(1); + expect(entries[0].detail).toBe('Source: email re: offer, signed'); + }); + + test('does not double-extract a timeline bullet carrying its own citation', () => { + const entries = parseTimelineEntries('- **2025-03-18** | Meeting notes [Source: notes, 2025-03-18]'); + expect(entries).toHaveLength(1); // bullet pass only + }); + + test('skips invalid calendar dates and bare citations', () => { + expect(parseTimelineEntries('Claim. [Source: memo, 2026-13-45]')).toHaveLength(0); + expect(parseTimelineEntries('[Source: import batch, 2025-07-01]')).toHaveLength(0); + }); +});