mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-31 04:07:52 +00:00
feat(extract): --infer-dates anchors timeline from a page's content date when its body has none (#2341)
* feat(extract): --infer-dates anchors timeline from a page's content date when its body has none parseTimelineEntries only reads in-body date lines (`- **YYYY-MM-DD** | ...`). Comms- and calendar-dominated brains keep the date in frontmatter or the filename (slug `2026-04-24-...`), so those pages yield zero timeline entries and find_trajectory stays blind even though the page is firmly dated. `--infer-dates` (opt-in, DB-source) anchors ONE timeline entry at the page's already-computed `effective_date` for pages whose body parse returns nothing. Trustworthy sources only (frontmatter event_date/date/published or the filename date) — never the `updated_at` fallback. Applied solely on the zero-entry path so it can never shadow a real in-body timeline. - new pure helper `deriveTimelineAnchor()` in link-extraction.ts (+6 unit tests) - `getPage()` now projects effective_date/effective_date_source in BOTH engines (engine parity) - on a comms-heavy ~13.7K-page brain this lifts a dry-run timeline yield from 1 to 11,006 entries (timeline coverage 0% -> ~80%) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01E5wtDU4ZLKewXUYkPLQHSy * docs(extract): correct deriveTimelineAnchor comment — feeds page timeline, not find_trajectory find_trajectory reads the facts table by entity_slug; the page-level `timeline` table this helper populates feeds get_timeline + the brain-score timeline_coverage component instead. Comment-only; no behavior change. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01E5wtDU4ZLKewXUYkPLQHSy --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Garry Tan <garrytan@gmail.com> Co-authored-by: Time Attakc <89218912+time-attack@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Garry Tan
Time Attakc
parent
0413c93e72
commit
e9fa962929
+23
-4
@@ -35,7 +35,7 @@ import type { BrainEngine, LinkBatchInput, TimelineBatchInput } from '../core/en
|
||||
import type { PageType } from '../core/types.ts';
|
||||
import { parseMarkdown } from '../core/markdown.ts';
|
||||
import {
|
||||
extractPageLinks, parseTimelineEntries, inferLinkType, makeResolver,
|
||||
extractPageLinks, parseTimelineEntries, deriveTimelineAnchor, inferLinkType, makeResolver,
|
||||
extractFrontmatterLinks, isGlobalBasenameEnabled, LINK_EXTRACTOR_VERSION_TS,
|
||||
WIKILINK_BASENAME_LINK_TYPE,
|
||||
buildBasenameIndex, queryBasenameIndex, stripCodeBlocks,
|
||||
@@ -749,6 +749,12 @@ export async function runExtract(engine: BrainEngine, args: string[]) {
|
||||
// v0.41.18.0 (A11, T8): --from-meetings extracts timeline entries from
|
||||
// meeting pages onto each discussed entity. Timeline subcommand only.
|
||||
const fromMeetings = args.includes('--from-meetings');
|
||||
// --infer-dates: for pages whose body has NO parseable timeline line, anchor
|
||||
// one entry at the page's computed effective_date (frontmatter / filename date,
|
||||
// never the updated_at fallback). Default OFF for back-compat — comms/calendar
|
||||
// brains opt in to populate timeline from slug/frontmatter dates. DB-source only
|
||||
// (needs the full Page.effective_date, which getPage projects).
|
||||
const inferDates = args.includes('--infer-dates');
|
||||
// v0.41.17.0 (T7, D9): --workers N parsed via the shared validator.
|
||||
// Honored on the fs-walk inner loops only; DB-source paths stay
|
||||
// serial in v0.41.17.0 (see ExtractOpts.workers doc).
|
||||
@@ -963,7 +969,7 @@ Status (v0.42):
|
||||
result.pages_processed = r.pages;
|
||||
}
|
||||
if (subcommand === 'timeline' || subcommand === 'all') {
|
||||
const r = await extractTimelineFromDB(engine, dryRun, jsonMode, typeFilter, since, { sourceIdFilter });
|
||||
const r = await extractTimelineFromDB(engine, dryRun, jsonMode, typeFilter, since, { sourceIdFilter, inferDates });
|
||||
result.timeline_entries_created = r.created;
|
||||
result.pages_processed = Math.max(result.pages_processed, r.pages);
|
||||
}
|
||||
@@ -1583,7 +1589,7 @@ async function extractTimelineFromDB(
|
||||
jsonMode: boolean,
|
||||
typeFilter: PageType | undefined,
|
||||
since: string | undefined,
|
||||
opts?: { sourceIdFilter?: string },
|
||||
opts?: { sourceIdFilter?: string; inferDates?: boolean },
|
||||
): Promise<{ created: number; pages: number }> {
|
||||
// v0.32.8: listAllPageRefs enumerates (slug, source_id) pairs so we can
|
||||
// thread sourceId to getPage and addTimelineEntriesBatch. Pre-fix used
|
||||
@@ -1592,6 +1598,7 @@ async function extractTimelineFromDB(
|
||||
// v0.37.7.0 #1204: when sourceIdFilter is set, scope the walk to one
|
||||
// source so federated brain users can extract per-source.
|
||||
const sourceIdFilter = opts?.sourceIdFilter;
|
||||
const inferDates = opts?.inferDates ?? false;
|
||||
const allRefs = sourceIdFilter
|
||||
? (await engine.listAllPageRefs()).filter(r => r.source_id === sourceIdFilter)
|
||||
: await engine.listAllPageRefs();
|
||||
@@ -1631,7 +1638,19 @@ async function extractTimelineFromDB(
|
||||
}
|
||||
|
||||
const fullContent = page.compiled_truth + '\n' + page.timeline;
|
||||
const entries = parseTimelineEntries(fullContent);
|
||||
let entries = parseTimelineEntries(fullContent);
|
||||
// --infer-dates: pages with no in-body timeline line but a trustworthy
|
||||
// content date (frontmatter / filename) get one anchor entry at that date.
|
||||
// Applied ONLY on the zero-entry path so it never shadows a real timeline.
|
||||
if (entries.length === 0 && inferDates) {
|
||||
const anchor = deriveTimelineAnchor({
|
||||
slug,
|
||||
title: page.title,
|
||||
effectiveDate: page.effective_date,
|
||||
effectiveDateSource: page.effective_date_source,
|
||||
});
|
||||
if (anchor) entries = [anchor];
|
||||
}
|
||||
|
||||
for (const entry of entries) {
|
||||
if (dryRunSeen) {
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
*/
|
||||
|
||||
import type { BrainEngine } from './engine.ts';
|
||||
import type { PageType } from './types.ts';
|
||||
import type { PageType, EffectiveDateSource } from './types.ts';
|
||||
import { ensureWellFormed } from './text-safe.ts';
|
||||
|
||||
/**
|
||||
@@ -1290,6 +1290,46 @@ function isValidDate(s: string): boolean {
|
||||
return dt.getUTCFullYear() === y && dt.getUTCMonth() === mo - 1 && dt.getUTCDate() === d;
|
||||
}
|
||||
|
||||
/** Input for {@link deriveTimelineAnchor}: a page's identity + its computed content date. */
|
||||
export interface TimelineAnchorInput {
|
||||
slug: string;
|
||||
title?: string | null;
|
||||
effectiveDate?: Date | string | null;
|
||||
effectiveDateSource?: EffectiveDateSource | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Anchor a single timeline entry from a page's computed content date, for pages
|
||||
* whose body carries no parseable timeline line.
|
||||
*
|
||||
* Comms- and calendar-dominated brains keep the date in frontmatter or the
|
||||
* filename (slug `2026-04-24-...`), not in the prose, so `parseTimelineEntries`
|
||||
* returns nothing and the page-level `timeline` table stays empty even though
|
||||
* the page is firmly dated — leaving `get_timeline` and the brain-score
|
||||
* `timeline_coverage` component blind to it. This recovers that signal from the
|
||||
* already-computed `effective_date` (no re-parsing). (It does NOT feed the
|
||||
* facts-based `find_trajectory`, which reads the `facts` table by entity_slug.)
|
||||
*
|
||||
* Fires ONLY for a trustworthy content date — frontmatter (`event_date` / `date`
|
||||
* / `published`) or the `filename` date — never the `fallback` source, which is
|
||||
* `updated_at` (link-churn noise, not when the thing happened). Returns null
|
||||
* when no trustworthy date is available. Callers MUST apply this only when body
|
||||
* parsing yields zero entries, so it can never shadow a real in-body timeline.
|
||||
*/
|
||||
export function deriveTimelineAnchor(input: TimelineAnchorInput): TimelineCandidate | null {
|
||||
const { slug, title, effectiveDate, effectiveDateSource } = input;
|
||||
if (!effectiveDate) return null;
|
||||
// 'fallback' === updated_at; the rest ('event_date'|'date'|'published'|'filename')
|
||||
// are real content dates. null/undefined source is not trustworthy either.
|
||||
if (effectiveDateSource == null || effectiveDateSource === 'fallback') return null;
|
||||
const dt = typeof effectiveDate === 'string' ? new Date(effectiveDate) : effectiveDate;
|
||||
if (!(dt instanceof Date) || Number.isNaN(dt.getTime())) return null;
|
||||
const iso = dt.toISOString().slice(0, 10);
|
||||
if (!isValidDate(iso)) return null;
|
||||
const summary = (title ?? '').trim() || slug.split('/').pop() || slug;
|
||||
return { date: iso, summary, detail: '' };
|
||||
}
|
||||
|
||||
// ─── Auto-link config ───────────────────────────────────────────
|
||||
|
||||
/**
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
inferLinkType,
|
||||
makeResolver,
|
||||
parseTimelineEntries,
|
||||
deriveTimelineAnchor,
|
||||
isAutoLinkEnabled,
|
||||
FRONTMATTER_LINK_MAP,
|
||||
unwrapWikilink,
|
||||
@@ -852,6 +853,55 @@ More prose here.
|
||||
});
|
||||
});
|
||||
|
||||
// ─── deriveTimelineAnchor ──────────────────────────────────────
|
||||
|
||||
describe('deriveTimelineAnchor', () => {
|
||||
test('anchors at a frontmatter effective_date with the page title as summary', () => {
|
||||
const a = deriveTimelineAnchor({
|
||||
slug: 'meetings/2026-04-24-handover',
|
||||
title: 'Ops handover',
|
||||
effectiveDate: new Date('2026-04-24T09:00:00Z'),
|
||||
effectiveDateSource: 'event_date',
|
||||
});
|
||||
expect(a).toEqual({ date: '2026-04-24', summary: 'Ops handover', detail: '' });
|
||||
});
|
||||
|
||||
test('accepts a filename-sourced date and an ISO-string effectiveDate', () => {
|
||||
const a = deriveTimelineAnchor({
|
||||
slug: 'daily/2022-04-20-standup',
|
||||
title: '',
|
||||
effectiveDate: '2022-04-20',
|
||||
effectiveDateSource: 'filename',
|
||||
});
|
||||
expect(a).toEqual({ date: '2022-04-20', summary: '2022-04-20-standup', detail: '' });
|
||||
});
|
||||
|
||||
test('returns null for the fallback (updated_at) source — not a real content date', () => {
|
||||
expect(deriveTimelineAnchor({
|
||||
slug: 'notes/x', title: 'X',
|
||||
effectiveDate: new Date('2026-01-01T00:00:00Z'),
|
||||
effectiveDateSource: 'fallback',
|
||||
})).toBeNull();
|
||||
});
|
||||
|
||||
test('returns null when no date or no source', () => {
|
||||
expect(deriveTimelineAnchor({ slug: 'a', effectiveDate: null, effectiveDateSource: 'date' })).toBeNull();
|
||||
expect(deriveTimelineAnchor({ slug: 'a', effectiveDate: new Date('2026-01-01Z'), effectiveDateSource: null })).toBeNull();
|
||||
});
|
||||
|
||||
test('returns null on an unparseable date string', () => {
|
||||
expect(deriveTimelineAnchor({ slug: 'a', title: 'A', effectiveDate: 'not-a-date', effectiveDateSource: 'date' })).toBeNull();
|
||||
});
|
||||
|
||||
test('falls back to the slug basename when title is empty', () => {
|
||||
const a = deriveTimelineAnchor({
|
||||
slug: 'people/jane-example-com', title: ' ',
|
||||
effectiveDate: '2025-12-31', effectiveDateSource: 'published',
|
||||
});
|
||||
expect(a?.summary).toBe('jane-example-com');
|
||||
});
|
||||
});
|
||||
|
||||
// ─── isAutoLinkEnabled ─────────────────────────────────────────
|
||||
|
||||
function makeFakeEngine(configMap: Map<string, string | null>): BrainEngine {
|
||||
|
||||
Reference in New Issue
Block a user