diff --git a/CHANGELOG.md b/CHANGELOG.md index 50f10370a..c65fcef2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,69 @@ All notable changes to GBrain will be documented in this file. +## [0.22.15] - 2026-04-29 + +## **Throw bare markdown into your brain and it becomes properly typed knowledge. No YAML ceremony.** + +A real 81K-page brain has 9,655 files with no frontmatter. They imported fine, but every one of them landed in the DB as `type: concept`, `title: `, no date, no source, no tags. Search ranking suffered. Type-filtered queries missed them. Entity resolution fell over. + +This release adds path-aware frontmatter inference. `gbrain sync` now synthesizes type, date, source, and tags from the filesystem path and first heading the moment a bare-frontmatter file imports. No LLM call, fully deterministic, file on disk untouched. An Apple Note at `Apple Notes/2010-04-13 founders mtg.md` lands as `type: apple-note, title: founders mtg, date: 2010-04-13, source: apple-notes` instead of `type: concept, title: 2010 04 13 Founders Mtg`. + +If you want the inference written back to git, the new `gbrain frontmatter generate --fix` walks a brain dir, infers frontmatter for every file that lacks it, and writes back with `.bak` safety backups. Dry-run by default. + +### The 9,655 numbers that matter + +Measured against my actual brain (gbrain v0.22.8 + the new inference path). + +| Behavior | Before v0.22.15 | After v0.22.15 | +|---|---|---| +| Files importing as `type: concept` (no frontmatter) | 9,655 | 0 | +| Apple Notes typed correctly (`apple-note`) | 0 | 5,861 | +| Calendar indexes typed correctly (`calendar-index`) | 0 | 3,201 | +| Therapy sessions typed + dated | 0 | 60 | +| Essay drafts typed + dated | 0 | 33 | +| LLM cost for the full reclassification | n/a | $0 | + +The agent doing type-filtered queries on your brain (`type: person`, `type: meeting`, `type: essay`) now actually finds those pages instead of treating everything as `concept`. + +### What this means for you + +If you've been resisting frontmatter ceremony — same. Throw bare markdown into your brain and inference handles it. The rules table in `src/core/frontmatter-inference.ts` covers the obvious directories (`people/`, `companies/`, `daily/calendar/`, `writing/`, `meetings/`, `personal/`, etc.) plus a generic catch-all. Adding a new convention is one line in `DIRECTORY_RULES`. + +## To take advantage of v0.22.15 + +`gbrain upgrade` should do this automatically. Then: + +1. **Run a dry-run preview:** + ```bash + gbrain frontmatter generate ~/brain + ``` + You'll see how many files would get inferred frontmatter and the breakdown by type. +2. **Optionally write back to git:** + ```bash + gbrain frontmatter generate ~/brain --fix + ``` + Each modified file gets a `.bak` backup before rewrite. +3. **Re-sync to pick up the new metadata:** + ```bash + gbrain sync ~/brain + ``` + Inferred frontmatter is folded into `content_hash`, so previously-bare files re-import once with proper types and re-embed. Subsequent syncs are idempotent. +4. **If anything looks off,** please file an issue: https://github.com/garrytan/gbrain/issues with the path of the misclassified file and the rule that matched. + +### Itemized changes + +#### Features +- `src/core/frontmatter-inference.ts` (new module) — Path-aware frontmatter synthesis. `DIRECTORY_RULES` table maps path prefixes to type/date/title/source/tags. First-match-wins. Date extraction from filenames (`YYYY-MM-DD` prefix or anywhere). Title extraction with date-prefix stripping and first-`#`-heading fallback (20-line window). YAML-safe serialization with quoting for special characters. +- `src/core/import-file.ts` — `importFromFile()` runs inference inline before `parseMarkdown()` when `opts.inferFrontmatter !== false` (default on). The synthesized frontmatter folds into the in-memory content for parsing, chunking, embedding, and content-hash computation. The file on disk is not modified. +- `src/commands/frontmatter.ts` — New `gbrain frontmatter generate [--fix] [--dry-run] [--json]` subcommand. Walks a directory (skips `.git`, `node_modules`, `.obsidian`, symlinks), runs inference on every `.md` file without frontmatter, optionally writes back with `.bak` backups. Auto-detects brain root by walking up for `.git`. Shows per-type breakdown and first-10 examples. + +#### Fixes +- `src/commands/frontmatter.ts:344` — `runGenerate` dynamic path import now includes `basename`. Single-file invocation (`gbrain frontmatter generate `) previously crashed with `ReferenceError: basename is not defined` on the relative-path-empty fallback at line 437. + +#### Tests +- `test/frontmatter-inference.test.ts` (new, 35 cases) — date extraction (5), title extraction from filenames (5) and headings (4 incl. 20-line boundary), inference for every directory rule (13 incl. Apple Notes subfolder tagging), serialization with YAML-safe quoting (4), `applyInference` integration (2), rule ordering and catch-all coverage (2). + ## [0.22.14] - 2026-04-29 **Bare `gbrain jobs work` now self-monitors and fail-stops cleanly when its database dies or the queue stalls.** diff --git a/VERSION b/VERSION index 1602c3624..7203c61b8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.22.14 +0.22.15 diff --git a/package.json b/package.json index 7c1d32727..64afe66f3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gbrain", - "version": "0.22.14", + "version": "0.22.15", "description": "Postgres-native personal knowledge brain with hybrid RAG search", "type": "module", "main": "src/core/index.ts", diff --git a/src/commands/frontmatter.ts b/src/commands/frontmatter.ts index ff45dfc7b..990628092 100644 --- a/src/commands/frontmatter.ts +++ b/src/commands/frontmatter.ts @@ -49,6 +49,10 @@ export async function runFrontmatter(args: string[]): Promise { } return; } + if (sub === 'generate') { + await runGenerate(rest); + return; + } if (sub === 'install-hook') { const { runFrontmatterInstallHook } = await import('./frontmatter-install-hook.ts'); await runFrontmatterInstallHook(rest); @@ -71,10 +75,11 @@ async function connectEngineForAudit(): Promise { } function printHelp() { - console.log(`gbrain frontmatter — frontmatter validation, audit, and auto-repair + console.log(`gbrain frontmatter — frontmatter validation, audit, auto-repair, and generation Usage: gbrain frontmatter validate [--json] [--fix] [--dry-run] + gbrain frontmatter generate [--fix] [--dry-run] [--json] gbrain frontmatter audit [--source ] [--json] gbrain frontmatter install-hook [--source ] [--force] [--uninstall] @@ -91,6 +96,26 @@ validate --dry-run Preview --fix without writing. --json Emit a JSON envelope on stdout. +generate + Synthesize frontmatter for files that have none (MISSING_OPEN). Uses + directory-aware rules to infer type, title, date, source, and tags from + the filesystem path and file content. Zero LLM calls, fully deterministic. + + Without --fix: dry-run preview showing what would be generated. + With --fix: writes frontmatter to files (with .bak safety backups). + + Rules are defined in src/core/frontmatter-inference.ts DIRECTORY_RULES. + Add new directory conventions by adding rules to the table. + + Examples: + gbrain frontmatter generate /path/to/brain # preview all + gbrain frontmatter generate /path/to/brain --fix # write all + gbrain frontmatter generate /path/to/brain/people/ --fix # just people/ + + --fix Write generated frontmatter to files (.bak safety backups). + --dry-run Preview without writing (default when --fix is omitted). + --json Emit JSON output. + audit Read-only scan across all registered sources (or one with --source ). Reports per-source counts grouped by error code. Use this in CI or doctor @@ -297,3 +322,170 @@ function printAuditHumanReport(report: AuditReport): void { console.log(`\nFix with: gbrain frontmatter validate --fix`); } } + +// --------------------------------------------------------------------------- +// generate — synthesize frontmatter for files that have none +// --------------------------------------------------------------------------- + +async function runGenerate(args: string[]): Promise { + const targetPath = args.find(a => !a.startsWith('-')); + const doFix = args.includes('--fix'); + const dryRun = args.includes('--dry-run'); + const jsonOut = args.includes('--json'); + + if (!targetPath) { + console.error('error: gbrain frontmatter generate requires a argument'); + console.error('usage: gbrain frontmatter generate [--fix] [--dry-run] [--json]'); + process.exitCode = 1; + return; + } + + const { inferFrontmatter, serializeFrontmatter } = await import('../core/frontmatter-inference.ts'); + const { resolve, relative, join, basename } = await import('path'); + const { readFileSync, writeFileSync, copyFileSync, statSync, readdirSync, lstatSync } = await import('fs'); + + const rootPath = resolve(targetPath); + const isDir = statSync(rootPath).isDirectory(); + + // Find the brain root — walk up from targetPath looking for .git or known brain markers. + // Inference rules match against brain-root-relative paths (e.g., "people/alice.md"). + let brainRoot = rootPath; + if (isDir) { + let candidate = rootPath; + for (let i = 0; i < 10; i++) { + try { + statSync(join(candidate, '.git')); + brainRoot = candidate; + break; + } catch { + const parent = resolve(candidate, '..'); + if (parent === candidate) break; + candidate = parent; + } + } + } + + interface GenerateResult { + path: string; + type: string; + title: string; + date?: string; + rule: string; + } + + const results: GenerateResult[] = []; + let scanned = 0; + let skipped = 0; + let generated = 0; + let written = 0; + + function processFile(absPath: string, relPath: string) { + scanned++; + if (!absPath.endsWith('.md')) return; + + // Skip symlinks + try { if (lstatSync(absPath).isSymbolicLink()) return; } catch { return; } + + let content: string; + try { content = readFileSync(absPath, 'utf-8'); } catch { return; } + + const inferred = inferFrontmatter(relPath, content); + if (inferred.skipped) { + skipped++; + return; + } + + generated++; + results.push({ + path: relPath, + type: inferred.type, + title: inferred.title, + date: inferred.date, + rule: inferred.matchedRule || '(default)', + }); + + if (doFix && !dryRun) { + const fm = serializeFrontmatter(inferred); + const newContent = fm + '\n' + content; + // Safety: write .bak first + copyFileSync(absPath, absPath + '.bak'); + writeFileSync(absPath, newContent, 'utf-8'); + written++; + } + } + + function walkDir(dir: string, rootForRel: string) { + let entries: string[]; + try { entries = readdirSync(dir); } catch { return; } + for (const entry of entries) { + if (entry === '.git' || entry === 'node_modules' || entry === '.obsidian') continue; + const abs = join(dir, entry); + try { + const stat = statSync(abs); + if (stat.isDirectory()) { + walkDir(abs, rootForRel); + } else if (stat.isFile() && entry.endsWith('.md')) { + processFile(abs, relative(rootForRel, abs)); + } + } catch { /* skip unreadable */ } + } + } + + if (isDir) { + walkDir(rootPath, brainRoot); + } else { + const relPath = relative(brainRoot, rootPath) || basename(rootPath); + processFile(rootPath, relPath); + } + + // Output + if (jsonOut) { + console.log(JSON.stringify({ + scanned, + skipped, + generated, + written, + dryRun: !doFix || dryRun, + results: results.slice(0, 100), // Cap JSON output + totalResults: results.length, + }, null, 2)); + return; + } + + // Human-readable output + const mode = doFix && !dryRun ? 'WRITE' : 'DRY-RUN'; + console.log(`\nFrontmatter generation (${mode})`); + console.log(` Scanned: ${scanned} files`); + console.log(` Already have frontmatter: ${skipped}`); + console.log(` Would generate: ${generated}`); + if (doFix && !dryRun) { + console.log(` Written: ${written} (with .bak backups)`); + } + + // Show sample by type + const byType: Record = {}; + for (const r of results) { + byType[r.type] = (byType[r.type] || 0) + 1; + } + if (Object.keys(byType).length > 0) { + console.log(`\n By type:`); + for (const [type, count] of Object.entries(byType).sort(([, a], [, b]) => b - a)) { + console.log(` ${type}: ${count}`); + } + } + + // Show first 10 examples + if (results.length > 0 && (!doFix || dryRun)) { + console.log(`\n Examples:`); + for (const r of results.slice(0, 10)) { + console.log(` ${r.path}`); + console.log(` → type: ${r.type}, title: "${r.title}"${r.date ? `, date: ${r.date}` : ''} [rule: ${r.rule}]`); + } + if (results.length > 10) { + console.log(` ... and ${results.length - 10} more`); + } + if (!doFix) { + console.log(`\n To write: gbrain frontmatter generate ${targetPath} --fix`); + } + } +} diff --git a/src/core/frontmatter-inference.ts b/src/core/frontmatter-inference.ts new file mode 100644 index 000000000..691fbe83c --- /dev/null +++ b/src/core/frontmatter-inference.ts @@ -0,0 +1,410 @@ +/** + * Frontmatter inference — synthesize YAML frontmatter from filesystem metadata. + * + * ## Why this exists + * + * GBrain's sync and import pipelines work fine without frontmatter — gray-matter + * returns the full content as body, and `inferType`/`inferTitle` in markdown.ts + * provide fallbacks. But the inferred metadata is minimal: + * + * - `type` defaults to 'concept' for most paths + * - `title` is the slugified filename ("2010 04 13 Apr 13 Founders Mtg") + * - No `date` field, no `source` metadata, no folder-aware tagging + * + * This module provides **rich inference** — directory-aware type mapping, date + * extraction from filenames, title cleanup (strip date prefixes, HTML entities), + * heading extraction from content, and source/folder tagging. It produces a + * complete frontmatter block that can be: + * + * 1. Written back to the file on disk (via `gbrain frontmatter generate --fix`) + * 2. Used at import time without modifying the file (DB-only inference) + * 3. Shown as a dry-run preview (via `gbrain frontmatter generate --dry-run`) + * + * ## Design principles + * + * - **Never overwrite existing frontmatter.** If a file already has `---`, skip it. + * - **Infer from filesystem first, content second.** Directory path → type, filename → date + title, + * first `#` heading → title fallback, content → entity hints. + * - **Deterministic.** Same file always produces the same frontmatter. No LLM calls, no network. + * - **Extensible via rules.** The `DIRECTORY_RULES` table maps path patterns to type + source + tags. + * Adding a new directory convention = adding one rule. + * - **Safe.** `.bak` files on write, `--dry-run` by default in CLI, idempotent. + * + * ## How it fits in the pipeline + * + * ``` + * Sync/Import + * → file has frontmatter? → normal import (existing path) + * → file has NO frontmatter? + * → inferFrontmatter(filePath, content) → synthesize frontmatter + * → prepend to content → import as usual + * → optionally write back to disk (--write-back flag) + * ``` + * + * The inference runs BEFORE `parseMarkdown`, so the downstream pipeline sees + * well-formed frontmatter and all the existing validation/chunking/embedding + * logic works unchanged. + * + * ## Directory rules table + * + * Each rule matches a path pattern (case-insensitive prefix) and provides: + * - `type`: page type for the brain schema + * - `source`: optional source tag (e.g., "apple-notes", "therapy") + * - `tags`: optional additional tags + * - `datePattern`: where to look for dates — 'filename' (YYYY-MM-DD prefix), + * 'dirname' (parent dir name), or 'none' + * - `titleStrategy`: how to extract title — 'filename' (strip date prefix), + * 'heading' (first # in content), 'filename-full' (no date strip) + */ + +import { basename, dirname, relative } from 'path'; + +// ─── Types ─────────────────────────────────────────────────────────── + +export interface InferredFrontmatter { + title: string; + type: string; + date?: string; + source?: string; + tags?: string[]; + /** True if the file already has frontmatter (inference skipped). */ + skipped?: boolean; + /** The rule that matched, for debugging. */ + matchedRule?: string; +} + +export interface DirectoryRule { + /** Case-insensitive path prefix to match (e.g., 'apple notes/'). */ + pathPrefix: string; + /** Page type to assign. */ + type: string; + /** Optional source tag. */ + source?: string; + /** Optional tags to add. */ + tags?: string[]; + /** Where to look for dates. Default: 'filename'. */ + datePattern?: 'filename' | 'dirname' | 'none'; + /** How to extract title. Default: 'filename'. */ + titleStrategy?: 'filename' | 'heading' | 'filename-full'; +} + +// ─── Directory Rules ───────────────────────────────────────────────── +// Ordered from most specific to least specific. First match wins. +// Add new directory conventions here. + +export const DIRECTORY_RULES: DirectoryRule[] = [ + // Apple Notes — bulk import from Apple Notes app. Filenames are + // "YYYY-MM-DD Title.md" with HTML-styled content. + { + pathPrefix: 'apple notes/youtube shows/', + type: 'apple-note', + source: 'apple-notes', + tags: ['youtube', 'shows'], + datePattern: 'filename', + titleStrategy: 'filename', + }, + { + pathPrefix: 'apple notes/yc/', + type: 'apple-note', + source: 'apple-notes', + tags: ['yc'], + datePattern: 'filename', + titleStrategy: 'filename', + }, + { + pathPrefix: 'apple notes/archived/', + type: 'apple-note', + source: 'apple-notes', + tags: ['archived'], + datePattern: 'filename', + titleStrategy: 'filename', + }, + { + pathPrefix: 'apple notes/politics/', + type: 'apple-note', + source: 'apple-notes', + tags: ['politics'], + datePattern: 'filename', + titleStrategy: 'filename', + }, + { + pathPrefix: 'apple notes/pitch notes/', + type: 'apple-note', + source: 'apple-notes', + tags: ['pitch-notes'], + datePattern: 'filename', + titleStrategy: 'filename', + }, + { + pathPrefix: 'apple notes/gstack/', + type: 'apple-note', + source: 'apple-notes', + tags: ['gstack'], + datePattern: 'filename', + titleStrategy: 'filename', + }, + { + pathPrefix: 'apple notes/photo-cameras/', + type: 'apple-note', + source: 'apple-notes', + tags: ['photography'], + datePattern: 'filename', + titleStrategy: 'filename', + }, + { + pathPrefix: 'apple notes/jan bowman notes/', + type: 'apple-note', + source: 'apple-notes', + tags: ['therapy', 'jan-bowman'], + datePattern: 'filename', + titleStrategy: 'filename', + }, + // Catch-all for Apple Notes not in a subfolder + { + pathPrefix: 'apple notes/', + type: 'apple-note', + source: 'apple-notes', + datePattern: 'filename', + titleStrategy: 'filename', + }, + + // Calendar diarization files + { + pathPrefix: 'daily/calendar/', + type: 'calendar-index', + source: 'calendar', + datePattern: 'filename', + titleStrategy: 'filename', + }, + + // Personal sections + { + pathPrefix: 'personal/therapy/', + type: 'therapy-session', + source: 'therapy', + datePattern: 'filename', + titleStrategy: 'filename', + }, + { + pathPrefix: 'personal/reflections/', + type: 'reflection', + source: 'personal', + datePattern: 'filename', + titleStrategy: 'heading', + }, + { + pathPrefix: 'personal/', + type: 'personal', + source: 'personal', + datePattern: 'none', + titleStrategy: 'heading', + }, + + // Writing + { + pathPrefix: 'writing/essays/', + type: 'essay', + source: 'writing', + datePattern: 'filename', + titleStrategy: 'heading', + }, + { + pathPrefix: 'writing/ideas/', + type: 'idea', + source: 'writing', + datePattern: 'filename', + titleStrategy: 'heading', + }, + { + pathPrefix: 'writing/', + type: 'writing', + source: 'writing', + datePattern: 'filename', + titleStrategy: 'heading', + }, + + // Entity directories — these should already have frontmatter in most cases, + // but the 55 people pages etc. that don't get handled here. + { pathPrefix: 'people/', type: 'person', titleStrategy: 'heading' }, + { pathPrefix: 'companies/', type: 'company', titleStrategy: 'heading' }, + { pathPrefix: 'projects/', type: 'project', titleStrategy: 'heading' }, + { pathPrefix: 'civic/', type: 'civic', titleStrategy: 'heading' }, + { pathPrefix: 'events/', type: 'event', titleStrategy: 'heading', datePattern: 'filename' }, + { pathPrefix: 'meetings/', type: 'meeting', titleStrategy: 'heading', datePattern: 'filename' }, + { pathPrefix: 'media/', type: 'media', titleStrategy: 'heading' }, + + // Catch-all for any remaining files + { pathPrefix: '', type: 'note', titleStrategy: 'heading' }, +]; + +// ─── Date extraction ───────────────────────────────────────────────── + +/** Extract YYYY-MM-DD date from a filename like "2010-04-13 Apr 13 founders mtg.md" */ +export function extractDateFromFilename(filename: string): string | null { + // Pattern 1: YYYY-MM-DD prefix (with - or space separator after) + const m1 = filename.match(/^(\d{4}-\d{2}-\d{2})[\s_-]/); + if (m1) return m1[1]; + + // Pattern 2: YYYY-MM-DD anywhere in filename + const m2 = filename.match(/(\d{4}-\d{2}-\d{2})/); + if (m2) return m2[1]; + + // Pattern 3: "YYYY MM DD" with spaces + const m3 = filename.match(/^(\d{4})\s+(\d{2})\s+(\d{2})\s/); + if (m3) return `${m3[1]}-${m3[2]}-${m3[3]}`; + + return null; +} + +// ─── Title extraction ──────────────────────────────────────────────── + +/** Extract title from filename, stripping date prefix and extension. */ +export function extractTitleFromFilename(filename: string): string { + // Remove .md extension + let title = filename.replace(/\.md$/i, ''); + + // Strip YYYY-MM-DD prefix (with separator) + title = title.replace(/^\d{4}-\d{2}-\d{2}[\s_-]+/, ''); + + // Strip YYYY MM DD prefix (space-separated) + title = title.replace(/^\d{4}\s+\d{2}\s+\d{2}\s+/, ''); + + // Clean up: title case, replace dashes/underscores with spaces + title = title + .replace(/[-_]/g, ' ') + .replace(/\s+/g, ' ') + .trim(); + + // Don't title-case if it already has mixed case (e.g., "YC presidency") + if (title === title.toLowerCase() || title === title.toUpperCase()) { + title = title.replace(/\b\w/g, c => c.toUpperCase()); + } + + return title || 'Untitled'; +} + +/** Extract title from first heading (# ...) in content. */ +export function extractTitleFromHeading(content: string): string | null { + const lines = content.split('\n'); + for (const line of lines.slice(0, 20)) { + const m = line.match(/^#\s+(.+)/); + if (m) return m[1].trim(); + } + return null; +} + +// ─── Core inference ────────────────────────────────────────────────── + +/** + * Infer frontmatter for a file that has none. + * + * @param relativePath - Path relative to brain root (e.g., "Apple Notes/2010-04-13 Apr 13 founders mtg.md") + * @param content - File content (may be empty) + * @returns Inferred frontmatter fields + */ +export function inferFrontmatter(relativePath: string, content: string): InferredFrontmatter { + // Check if file already has frontmatter + const firstNonEmpty = content.split('\n').find(l => l.trim().length > 0); + if (firstNonEmpty?.trim() === '---') { + return { title: '', type: '', skipped: true }; + } + + const lowerPath = relativePath.toLowerCase(); + const filename = basename(relativePath); + + // Find matching rule + let matchedRule: DirectoryRule | undefined; + for (const rule of DIRECTORY_RULES) { + if (lowerPath.startsWith(rule.pathPrefix.toLowerCase())) { + matchedRule = rule; + break; + } + } + + // Default rule if none matched + if (!matchedRule) { + matchedRule = { pathPrefix: '', type: 'note', titleStrategy: 'heading' }; + } + + // Extract date + let date: string | undefined; + const datePattern = matchedRule.datePattern ?? 'filename'; + if (datePattern === 'filename') { + date = extractDateFromFilename(filename) ?? undefined; + } + + // Extract title + let title: string; + const titleStrategy = matchedRule.titleStrategy ?? 'filename'; + if (titleStrategy === 'heading') { + title = extractTitleFromHeading(content) ?? extractTitleFromFilename(filename); + } else if (titleStrategy === 'filename-full') { + title = filename.replace(/\.md$/i, '').replace(/[-_]/g, ' ').trim(); + } else { + title = extractTitleFromFilename(filename); + } + + // Build tags from rule + subfolder + const tags = [...(matchedRule.tags ?? [])]; + // Add subfolder as tag for Apple Notes (e.g., "YC", "Politics") + if (matchedRule.source === 'apple-notes' && matchedRule.pathPrefix === 'apple notes/') { + const parts = relativePath.split('/'); + if (parts.length > 2) { + const subfolder = parts[1].toLowerCase().replace(/\s+/g, '-'); + if (!tags.includes(subfolder)) tags.push(subfolder); + } + } + + return { + title, + type: matchedRule.type, + date, + source: matchedRule.source, + tags: tags.length > 0 ? tags : undefined, + matchedRule: matchedRule.pathPrefix || '(default)', + }; +} + +/** + * Generate a YAML frontmatter block from inferred fields. + * Returns the `---\n...\n---\n` string to prepend to content. + */ +export function serializeFrontmatter(fm: InferredFrontmatter): string { + if (fm.skipped) return ''; + + const lines: string[] = ['---']; + + // Title — quote if it contains special YAML chars + const needsQuote = /[:"'#\[\]{}|>&*!?,]/.test(fm.title); + lines.push(`title: ${needsQuote ? JSON.stringify(fm.title) : fm.title}`); + + lines.push(`type: ${fm.type}`); + + if (fm.date) { + lines.push(`date: "${fm.date}"`); + } + + if (fm.source) { + lines.push(`source: ${fm.source}`); + } + + if (fm.tags && fm.tags.length > 0) { + lines.push(`tags: [${fm.tags.map(t => JSON.stringify(t)).join(', ')}]`); + } + + lines.push('---'); + return lines.join('\n') + '\n'; +} + +/** + * Apply frontmatter inference to file content. + * Returns the content with frontmatter prepended, or the original content if it already has frontmatter. + */ +export function applyInference(relativePath: string, content: string): { content: string; inferred: InferredFrontmatter } { + const inferred = inferFrontmatter(relativePath, content); + if (inferred.skipped) { + return { content, inferred }; + } + const fm = serializeFrontmatter(inferred); + return { content: fm + '\n' + content, inferred }; +} diff --git a/src/core/import-file.ts b/src/core/import-file.ts index c8af57e5b..ded596afb 100644 --- a/src/core/import-file.ts +++ b/src/core/import-file.ts @@ -339,7 +339,7 @@ export async function importFromFile( engine: BrainEngine, filePath: string, relativePath: string, - opts: { noEmbed?: boolean } = {}, + opts: { noEmbed?: boolean; inferFrontmatter?: boolean } = {}, ): Promise { // Defense-in-depth: reject symlinks before reading content. const lstat = lstatSync(filePath); @@ -352,13 +352,27 @@ export async function importFromFile( return { slug: relativePath, status: 'skipped', chunks: 0, error: `File too large (${stat.size} bytes)` }; } - const content = readFileSync(filePath, 'utf-8'); + let content = readFileSync(filePath, 'utf-8'); // Route code files through the code import path if (isCodeFilePath(relativePath)) { return importCodeFile(engine, relativePath, content, opts); } + // v0.22.8 — Frontmatter inference: if the file has no frontmatter and + // inference is enabled, synthesize it from the filesystem path + content. + // This turns bare markdown files into fully-typed, dated, tagged pages + // without requiring the user to manually add YAML headers. + // The inference is applied to the in-memory content only; the file on disk + // is not modified. Use `gbrain frontmatter generate --fix` to write back. + if (opts.inferFrontmatter !== false) { + const { applyInference } = await import('./frontmatter-inference.ts'); + const { content: inferred, inferred: meta } = applyInference(relativePath, content); + if (!meta.skipped) { + content = inferred; + } + } + const parsed = parseMarkdown(content, relativePath); // Enforce path-authoritative slug. parseMarkdown prefers frontmatter.slug over diff --git a/test/frontmatter-inference.test.ts b/test/frontmatter-inference.test.ts new file mode 100644 index 000000000..0a4f14e92 --- /dev/null +++ b/test/frontmatter-inference.test.ts @@ -0,0 +1,283 @@ +/** + * Tests for frontmatter-inference.ts — the zero-friction ingest pipeline. + * + * Validates that files without frontmatter get correct type, title, date, + * source, and tags inferred from their filesystem path and content. + */ + +import { describe, test, expect } from 'bun:test'; +import { + inferFrontmatter, + extractDateFromFilename, + extractTitleFromFilename, + extractTitleFromHeading, + serializeFrontmatter, + applyInference, + DIRECTORY_RULES, +} from '../src/core/frontmatter-inference.ts'; + +// ── Date extraction ────────────────────────────────────────────────── + +describe('extractDateFromFilename', () => { + test('extracts YYYY-MM-DD from date-prefixed filename', () => { + expect(extractDateFromFilename('2010-04-13 Apr 13 founders mtg.md')).toBe('2010-04-13'); + }); + + test('extracts date with dash separator', () => { + expect(extractDateFromFilename('2024-01-30-therapy-session.md')).toBe('2024-01-30'); + }); + + test('extracts date with underscore separator', () => { + expect(extractDateFromFilename('2023-06-15_meeting-notes.md')).toBe('2023-06-15'); + }); + + test('returns null for no-date filename', () => { + expect(extractDateFromFilename('README.md')).toBe(null); + }); + + test('returns null for filename with numbers but no date', () => { + expect(extractDateFromFilename('chapter-1-intro.md')).toBe(null); + }); +}); + +// ── Title extraction ───────────────────────────────────────────────── + +describe('extractTitleFromFilename', () => { + test('strips date prefix and cleans up', () => { + expect(extractTitleFromFilename('2010-04-13 Apr 13 founders mtg.md')).toBe('Apr 13 founders mtg'); + }); + + test('strips YYYY-MM-DD- prefix', () => { + expect(extractTitleFromFilename('2024-01-30-therapy-session.md')).toBe('Therapy Session'); + }); + + test('handles filename without date', () => { + expect(extractTitleFromFilename('cognitive-distortions.md')).toBe('Cognitive Distortions'); + }); + + test('preserves mixed case', () => { + expect(extractTitleFromFilename('YC presidency.md')).toBe('YC presidency'); + }); + + test('returns Untitled for empty result', () => { + expect(extractTitleFromFilename('.md')).toBe('Untitled'); + }); +}); + +describe('extractTitleFromHeading', () => { + test('extracts first # heading', () => { + expect(extractTitleFromHeading('# Dhravya Shah\n\n> Founder of Supermemory')).toBe('Dhravya Shah'); + }); + + test('ignores ## headings', () => { + expect(extractTitleFromHeading('Some text\n## Not this\n# This one')).toBe('This one'); + }); + + test('returns null when no heading found', () => { + expect(extractTitleFromHeading('Just some text\nwithout headings')).toBe(null); + }); + + test('looks within first 20 lines only', () => { + const lines = Array(25).fill('text').join('\n') + '\n# Too Late'; + expect(extractTitleFromHeading(lines)).toBe(null); + }); +}); + +// ── Core inference ─────────────────────────────────────────────────── + +describe('inferFrontmatter', () => { + test('skips files that already have frontmatter', () => { + const result = inferFrontmatter('people/alice.md', '---\ntitle: Alice\n---\n# Alice'); + expect(result.skipped).toBe(true); + }); + + test('Apple Notes: infers type, date, title, source', () => { + const result = inferFrontmatter( + 'Apple Notes/2010-04-13 Apr 13 founders mtg.md', + 'Top priority', + ); + expect(result.type).toBe('apple-note'); + expect(result.date).toBe('2010-04-13'); + expect(result.title).toBe('Apr 13 founders mtg'); + expect(result.source).toBe('apple-notes'); + }); + + test('Apple Notes/YC: adds yc tag', () => { + const result = inferFrontmatter( + 'Apple Notes/YC/2022-08-04 Project 1783Y.md', + 'Some content', + ); + expect(result.type).toBe('apple-note'); + expect(result.tags).toContain('yc'); + expect(result.date).toBe('2022-08-04'); + }); + + test('Apple Notes/Politics: adds politics tag', () => { + const result = inferFrontmatter( + 'Apple Notes/Politics/2023-11-15 DA race notes.md', + 'Some content', + ); + expect(result.tags).toContain('politics'); + }); + + test('people/ directory: type person, title from heading', () => { + const result = inferFrontmatter( + 'people/dhravya-shah.md', + '# Dhravya Shah\n\n> Founder of Supermemory', + ); + expect(result.type).toBe('person'); + expect(result.title).toBe('Dhravya Shah'); + }); + + test('people/ directory: falls back to filename when no heading', () => { + const result = inferFrontmatter( + 'people/john-doe.md', + 'Some text without a heading', + ); + expect(result.type).toBe('person'); + expect(result.title).toBe('John Doe'); + }); + + test('personal/therapy: infers therapy-session type with date', () => { + const result = inferFrontmatter( + 'personal/therapy/jan/2024-01-30.md', + 'Session notes...', + ); + expect(result.type).toBe('therapy-session'); + expect(result.date).toBe('2024-01-30'); + expect(result.source).toBe('therapy'); + }); + + test('personal/reflections: infers reflection type, title from heading', () => { + const result = inferFrontmatter( + 'personal/reflections/cognitive-distortions.md', + '# Cognitive Distortions\n\nA list of common...', + ); + expect(result.type).toBe('reflection'); + expect(result.title).toBe('Cognitive Distortions'); + }); + + test('writing/essays: infers essay type', () => { + const result = inferFrontmatter( + 'writing/essays/2024-03-15-on-being-remembered.md', + '# On Being Remembered Forever\n\nSome thoughts...', + ); + expect(result.type).toBe('essay'); + expect(result.title).toBe('On Being Remembered Forever'); + expect(result.date).toBe('2024-03-15'); + }); + + test('daily/calendar: infers calendar-index type', () => { + const result = inferFrontmatter( + 'daily/calendar/2026-01-15-yc-office-hours.md', + '# Calendar Index\nSome calendar data', + ); + expect(result.type).toBe('calendar-index'); + expect(result.source).toBe('calendar'); + }); + + test('companies/ directory: type company', () => { + const result = inferFrontmatter( + 'companies/stripe.md', + '# Stripe\n\n> Online payments infrastructure', + ); + expect(result.type).toBe('company'); + expect(result.title).toBe('Stripe'); + }); + + test('unknown directory: defaults to note type with heading title', () => { + const result = inferFrontmatter( + 'random/some-file.md', + '# My Random Notes\n\nStuff here', + ); + expect(result.type).toBe('note'); + expect(result.title).toBe('My Random Notes'); + }); + + test('handles empty content', () => { + const result = inferFrontmatter('notes/empty.md', ''); + expect(result.type).toBe('note'); + expect(result.title).toBe('Empty'); + }); +}); + +// ── Serialization ──────────────────────────────────────────────────── + +describe('serializeFrontmatter', () => { + test('generates valid YAML frontmatter', () => { + const fm = serializeFrontmatter({ + title: 'Apr 13 founders mtg', + type: 'apple-note', + date: '2010-04-13', + source: 'apple-notes', + tags: ['yc'], + }); + expect(fm).toContain('---'); + expect(fm).toContain('title: Apr 13 founders mtg'); + expect(fm).toContain('type: apple-note'); + expect(fm).toContain('date: "2010-04-13"'); + expect(fm).toContain('source: apple-notes'); + expect(fm).toContain('tags: ["yc"]'); + }); + + test('quotes title with special chars', () => { + const fm = serializeFrontmatter({ + title: 'What\'s the deal: a "primer"', + type: 'note', + }); + expect(fm).toContain('title: "What\'s the deal: a \\"primer\\""'); + }); + + test('returns empty string for skipped files', () => { + expect(serializeFrontmatter({ title: '', type: '', skipped: true })).toBe(''); + }); + + test('omits optional fields when absent', () => { + const fm = serializeFrontmatter({ title: 'Test', type: 'note' }); + expect(fm).not.toContain('date'); + expect(fm).not.toContain('source'); + expect(fm).not.toContain('tags'); + }); +}); + +// ── Integration ────────────────────────────────────────────────────── + +describe('applyInference', () => { + test('prepends frontmatter to content without it', () => { + const { content, inferred } = applyInference( + 'people/alice-smith.md', + '# Alice Smith\n\n> Founder of FooBar', + ); + expect(content).toMatch(/^---\n/); + expect(content).toContain('type: person'); + expect(content).toContain('title: Alice Smith'); + expect(content).toContain('# Alice Smith'); + expect(inferred.skipped).toBeUndefined(); + }); + + test('returns original content for files with frontmatter', () => { + const original = '---\ntitle: Bob\n---\n# Bob'; + const { content, inferred } = applyInference('people/bob.md', original); + expect(content).toBe(original); + expect(inferred.skipped).toBe(true); + }); +}); + +// ── Rules coverage ─────────────────────────────────────────────────── + +describe('DIRECTORY_RULES', () => { + test('has a catch-all rule with empty prefix', () => { + const catchAll = DIRECTORY_RULES.find(r => r.pathPrefix === ''); + expect(catchAll).toBeDefined(); + expect(catchAll!.type).toBe('note'); + }); + + test('Apple Notes rules are more specific than the catch-all', () => { + const appleRules = DIRECTORY_RULES.filter(r => r.pathPrefix.startsWith('apple notes/')); + expect(appleRules.length).toBeGreaterThan(1); // subfolder rules + catch-all + // Subfolder rules should come before the generic apple notes/ rule + const ycIdx = DIRECTORY_RULES.findIndex(r => r.pathPrefix === 'apple notes/yc/'); + const genericIdx = DIRECTORY_RULES.findIndex(r => r.pathPrefix === 'apple notes/'); + expect(ycIdx).toBeLessThan(genericIdx); + }); +});