feat(extract): --infer-dates anchors timeline from a page's content date when its body has none (#2341)

* feat(extract): --infer-dates anchors timeline from a page's content date when its body has none

parseTimelineEntries only reads in-body date lines (`- **YYYY-MM-DD** | ...`).
Comms- and calendar-dominated brains keep the date in frontmatter or the
filename (slug `2026-04-24-...`), so those pages yield zero timeline entries
and find_trajectory stays blind even though the page is firmly dated.

`--infer-dates` (opt-in, DB-source) anchors ONE timeline entry at the page's
already-computed `effective_date` for pages whose body parse returns nothing.
Trustworthy sources only (frontmatter event_date/date/published or the filename
date) — never the `updated_at` fallback. Applied solely on the zero-entry path
so it can never shadow a real in-body timeline.

- new pure helper `deriveTimelineAnchor()` in link-extraction.ts (+6 unit tests)
- `getPage()` now projects effective_date/effective_date_source in BOTH engines
  (engine parity)
- on a comms-heavy ~13.7K-page brain this lifts a dry-run timeline yield from 1
  to 11,006 entries (timeline coverage 0% -> ~80%)

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01E5wtDU4ZLKewXUYkPLQHSy

* docs(extract): correct deriveTimelineAnchor comment — feeds page timeline, not find_trajectory

find_trajectory reads the facts table by entity_slug; the page-level `timeline`
table this helper populates feeds get_timeline + the brain-score timeline_coverage
component instead. Comment-only; no behavior change.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01E5wtDU4ZLKewXUYkPLQHSy

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Garry Tan <garrytan@gmail.com>
Co-authored-by: Time Attakc <89218912+time-attack@users.noreply.github.com>
This commit is contained in:
johnnymn3monic
2026-07-27 18:04:46 -07:00
committed by GitHub
co-authored by Claude Opus 4.8 Garry Tan Time Attakc
parent 0413c93e72
commit e9fa962929
3 changed files with 114 additions and 5 deletions
+23 -4
View File
@@ -35,7 +35,7 @@ import type { BrainEngine, LinkBatchInput, TimelineBatchInput } from '../core/en
import type { PageType } from '../core/types.ts';
import { parseMarkdown } from '../core/markdown.ts';
import {
extractPageLinks, parseTimelineEntries, inferLinkType, makeResolver,
extractPageLinks, parseTimelineEntries, deriveTimelineAnchor, inferLinkType, makeResolver,
extractFrontmatterLinks, isGlobalBasenameEnabled, LINK_EXTRACTOR_VERSION_TS,
WIKILINK_BASENAME_LINK_TYPE,
buildBasenameIndex, queryBasenameIndex, stripCodeBlocks,
@@ -749,6 +749,12 @@ export async function runExtract(engine: BrainEngine, args: string[]) {
// v0.41.18.0 (A11, T8): --from-meetings extracts timeline entries from
// meeting pages onto each discussed entity. Timeline subcommand only.
const fromMeetings = args.includes('--from-meetings');
// --infer-dates: for pages whose body has NO parseable timeline line, anchor
// one entry at the page's computed effective_date (frontmatter / filename date,
// never the updated_at fallback). Default OFF for back-compat — comms/calendar
// brains opt in to populate timeline from slug/frontmatter dates. DB-source only
// (needs the full Page.effective_date, which getPage projects).
const inferDates = args.includes('--infer-dates');
// v0.41.17.0 (T7, D9): --workers N parsed via the shared validator.
// Honored on the fs-walk inner loops only; DB-source paths stay
// serial in v0.41.17.0 (see ExtractOpts.workers doc).
@@ -963,7 +969,7 @@ Status (v0.42):
result.pages_processed = r.pages;
}
if (subcommand === 'timeline' || subcommand === 'all') {
const r = await extractTimelineFromDB(engine, dryRun, jsonMode, typeFilter, since, { sourceIdFilter });
const r = await extractTimelineFromDB(engine, dryRun, jsonMode, typeFilter, since, { sourceIdFilter, inferDates });
result.timeline_entries_created = r.created;
result.pages_processed = Math.max(result.pages_processed, r.pages);
}
@@ -1583,7 +1589,7 @@ async function extractTimelineFromDB(
jsonMode: boolean,
typeFilter: PageType | undefined,
since: string | undefined,
opts?: { sourceIdFilter?: string },
opts?: { sourceIdFilter?: string; inferDates?: boolean },
): Promise<{ created: number; pages: number }> {
// v0.32.8: listAllPageRefs enumerates (slug, source_id) pairs so we can
// thread sourceId to getPage and addTimelineEntriesBatch. Pre-fix used
@@ -1592,6 +1598,7 @@ async function extractTimelineFromDB(
// v0.37.7.0 #1204: when sourceIdFilter is set, scope the walk to one
// source so federated brain users can extract per-source.
const sourceIdFilter = opts?.sourceIdFilter;
const inferDates = opts?.inferDates ?? false;
const allRefs = sourceIdFilter
? (await engine.listAllPageRefs()).filter(r => r.source_id === sourceIdFilter)
: await engine.listAllPageRefs();
@@ -1631,7 +1638,19 @@ async function extractTimelineFromDB(
}
const fullContent = page.compiled_truth + '\n' + page.timeline;
const entries = parseTimelineEntries(fullContent);
let entries = parseTimelineEntries(fullContent);
// --infer-dates: pages with no in-body timeline line but a trustworthy
// content date (frontmatter / filename) get one anchor entry at that date.
// Applied ONLY on the zero-entry path so it never shadows a real timeline.
if (entries.length === 0 && inferDates) {
const anchor = deriveTimelineAnchor({
slug,
title: page.title,
effectiveDate: page.effective_date,
effectiveDateSource: page.effective_date_source,
});
if (anchor) entries = [anchor];
}
for (const entry of entries) {
if (dryRunSeen) {
+41 -1
View File
@@ -12,7 +12,7 @@
*/
import type { BrainEngine } from './engine.ts';
import type { PageType } from './types.ts';
import type { PageType, EffectiveDateSource } from './types.ts';
import { ensureWellFormed } from './text-safe.ts';
/**
@@ -1290,6 +1290,46 @@ function isValidDate(s: string): boolean {
return dt.getUTCFullYear() === y && dt.getUTCMonth() === mo - 1 && dt.getUTCDate() === d;
}
/** Input for {@link deriveTimelineAnchor}: a page's identity + its computed content date. */
export interface TimelineAnchorInput {
slug: string;
title?: string | null;
effectiveDate?: Date | string | null;
effectiveDateSource?: EffectiveDateSource | null;
}
/**
* Anchor a single timeline entry from a page's computed content date, for pages
* whose body carries no parseable timeline line.
*
* Comms- and calendar-dominated brains keep the date in frontmatter or the
* filename (slug `2026-04-24-...`), not in the prose, so `parseTimelineEntries`
* returns nothing and the page-level `timeline` table stays empty even though
* the page is firmly dated — leaving `get_timeline` and the brain-score
* `timeline_coverage` component blind to it. This recovers that signal from the
* already-computed `effective_date` (no re-parsing). (It does NOT feed the
* facts-based `find_trajectory`, which reads the `facts` table by entity_slug.)
*
* Fires ONLY for a trustworthy content date — frontmatter (`event_date` / `date`
* / `published`) or the `filename` date — never the `fallback` source, which is
* `updated_at` (link-churn noise, not when the thing happened). Returns null
* when no trustworthy date is available. Callers MUST apply this only when body
* parsing yields zero entries, so it can never shadow a real in-body timeline.
*/
export function deriveTimelineAnchor(input: TimelineAnchorInput): TimelineCandidate | null {
const { slug, title, effectiveDate, effectiveDateSource } = input;
if (!effectiveDate) return null;
// 'fallback' === updated_at; the rest ('event_date'|'date'|'published'|'filename')
// are real content dates. null/undefined source is not trustworthy either.
if (effectiveDateSource == null || effectiveDateSource === 'fallback') return null;
const dt = typeof effectiveDate === 'string' ? new Date(effectiveDate) : effectiveDate;
if (!(dt instanceof Date) || Number.isNaN(dt.getTime())) return null;
const iso = dt.toISOString().slice(0, 10);
if (!isValidDate(iso)) return null;
const summary = (title ?? '').trim() || slug.split('/').pop() || slug;
return { date: iso, summary, detail: '' };
}
// ─── Auto-link config ───────────────────────────────────────────
/**
+50
View File
@@ -7,6 +7,7 @@ import {
inferLinkType,
makeResolver,
parseTimelineEntries,
deriveTimelineAnchor,
isAutoLinkEnabled,
FRONTMATTER_LINK_MAP,
unwrapWikilink,
@@ -852,6 +853,55 @@ More prose here.
});
});
// ─── deriveTimelineAnchor ──────────────────────────────────────
describe('deriveTimelineAnchor', () => {
test('anchors at a frontmatter effective_date with the page title as summary', () => {
const a = deriveTimelineAnchor({
slug: 'meetings/2026-04-24-handover',
title: 'Ops handover',
effectiveDate: new Date('2026-04-24T09:00:00Z'),
effectiveDateSource: 'event_date',
});
expect(a).toEqual({ date: '2026-04-24', summary: 'Ops handover', detail: '' });
});
test('accepts a filename-sourced date and an ISO-string effectiveDate', () => {
const a = deriveTimelineAnchor({
slug: 'daily/2022-04-20-standup',
title: '',
effectiveDate: '2022-04-20',
effectiveDateSource: 'filename',
});
expect(a).toEqual({ date: '2022-04-20', summary: '2022-04-20-standup', detail: '' });
});
test('returns null for the fallback (updated_at) source — not a real content date', () => {
expect(deriveTimelineAnchor({
slug: 'notes/x', title: 'X',
effectiveDate: new Date('2026-01-01T00:00:00Z'),
effectiveDateSource: 'fallback',
})).toBeNull();
});
test('returns null when no date or no source', () => {
expect(deriveTimelineAnchor({ slug: 'a', effectiveDate: null, effectiveDateSource: 'date' })).toBeNull();
expect(deriveTimelineAnchor({ slug: 'a', effectiveDate: new Date('2026-01-01Z'), effectiveDateSource: null })).toBeNull();
});
test('returns null on an unparseable date string', () => {
expect(deriveTimelineAnchor({ slug: 'a', title: 'A', effectiveDate: 'not-a-date', effectiveDateSource: 'date' })).toBeNull();
});
test('falls back to the slug basename when title is empty', () => {
const a = deriveTimelineAnchor({
slug: 'people/jane-example-com', title: ' ',
effectiveDate: '2025-12-31', effectiveDateSource: 'published',
});
expect(a?.summary).toBe('jane-example-com');
});
});
// ─── isAutoLinkEnabled ─────────────────────────────────────────
function makeFakeEngine(configMap: Map<string, string | null>): BrainEngine {