diff --git a/CHANGELOG.md b/CHANGELOG.md index 96b9951c6..373d19145 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ All notable changes to GBrain will be documented in this file. +## [0.5.1] - 2026-04-10 + +### Fixed + +- **Apple Notes and files with spaces just work.** Paths like `Apple Notes/2017-05-03 ohmygreen.md` now auto-slugify to clean slugs (`apple-notes/2017-05-03-ohmygreen`). Spaces become hyphens, parens and special characters are stripped, accented characters normalize to ASCII. All 5,861+ Apple Notes files import cleanly without manual renaming. +- **Existing brains auto-migrate.** On first run after upgrade, a one-time migration renames all existing slugs with spaces or special characters to their clean form. Links are rewritten automatically. No manual cleanup needed. +- **Import and sync produce identical slugs.** Both pipelines now use the same `slugifyPath()` function, eliminating the mismatch where sync preserved case but import lowercased. + ## [0.5.0] - 2026-04-10 ### Added diff --git a/VERSION b/VERSION index 8f0916f76..4b9fcbec1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.5.0 +0.5.1 diff --git a/src/core/markdown.ts b/src/core/markdown.ts index bf2e102e2..239fe054c 100644 --- a/src/core/markdown.ts +++ b/src/core/markdown.ts @@ -1,5 +1,6 @@ import matter from 'gray-matter'; import type { PageType } from './types.ts'; +import { slugifyPath } from './sync.ts'; export interface ParsedMarkdown { frontmatter: Record; @@ -148,17 +149,7 @@ function inferTitle(filePath?: string): string { function inferSlug(filePath?: string): string { if (!filePath) return 'untitled'; - - // Remove leading path components that are just the import root - // Keep the type directory + filename structure - let slug = filePath - .replace(/\.md$/i, '') - .replace(/\\/g, '/'); - - // Remove leading ./ - if (slug.startsWith('./')) slug = slug.slice(2); - - return slug.toLowerCase(); + return slugifyPath(filePath); } function extractTags(frontmatter: Record): string[] { diff --git a/src/core/migrate.ts b/src/core/migrate.ts index cb60b494b..b82905856 100644 --- a/src/core/migrate.ts +++ b/src/core/migrate.ts @@ -1,4 +1,5 @@ import type { BrainEngine } from './engine.ts'; +import { slugifyPath } from './sync.ts'; /** * Schema migrations — run automatically on initSchema(). @@ -8,20 +9,45 @@ import type { BrainEngine } from './engine.ts'; * * Each migration runs in a transaction: if the SQL fails, the version stays * where it was and the next run retries cleanly. + * + * Migrations can also include a handler function for application-level logic + * (e.g., data transformations that need TypeScript, not just SQL). */ interface Migration { version: number; name: string; sql: string; + handler?: (engine: BrainEngine) => Promise; } // Migrations are embedded here, not loaded from files. // Add new migrations at the end. Never modify existing ones. const MIGRATIONS: Migration[] = [ // Version 1 is the baseline (schema.sql creates everything with IF NOT EXISTS). - // Future migrations go here: - // { version: 2, name: 'add_aliases', sql: `ALTER TABLE pages ADD COLUMN IF NOT EXISTS aliases TEXT[];` }, + { + version: 2, + name: 'slugify_existing_pages', + sql: '', + handler: async (engine) => { + const pages = await engine.listPages(); + let renamed = 0; + for (const page of pages) { + const newSlug = slugifyPath(page.slug); + if (newSlug !== page.slug) { + try { + await engine.updateSlug(page.slug, newSlug); + await engine.rewriteLinks(page.slug, newSlug); + renamed++; + } catch (e: unknown) { + const msg = e instanceof Error ? e.message : String(e); + console.error(` Warning: could not rename "${page.slug}" → "${newSlug}": ${msg}`); + } + } + } + if (renamed > 0) console.log(` Renamed ${renamed} slugs`); + }, + }, ]; export const LATEST_VERSION = MIGRATIONS.length > 0 @@ -35,17 +61,24 @@ export async function runMigrations(engine: BrainEngine): Promise<{ applied: num let applied = 0; for (const m of MIGRATIONS) { if (m.version > current) { - // Each migration is transactional - await engine.transaction(async (tx) => { - // Execute the migration SQL via raw connection - // We need to access the underlying sql connection - const eng = tx as any; - const sql = eng.sql || eng._sql; - if (sql) { - await sql.unsafe(m.sql); - await sql`UPDATE config SET value = ${String(m.version)} WHERE key = 'version'`; - } - }); + // SQL migration (transactional) + if (m.sql) { + await engine.transaction(async (tx) => { + const eng = tx as any; + const sql = eng.sql || eng._sql; + if (sql) { + await sql.unsafe(m.sql); + } + }); + } + + // Application-level handler (runs outside transaction for flexibility) + if (m.handler) { + await m.handler(engine); + } + + // Update version after both SQL and handler succeed + await engine.setConfig('version', String(m.version)); console.log(` Migration ${m.version} applied: ${m.name}`); applied++; } diff --git a/src/core/sync.ts b/src/core/sync.ts index 8da1a3a80..c37b3dbfd 100644 --- a/src/core/sync.ts +++ b/src/core/sync.ts @@ -97,21 +97,39 @@ export function isSyncable(path: string): boolean { } /** - * Convert a repo-relative file path to a GBrain page slug. + * Slugify a single path segment: lowercase, strip special chars, spaces → hyphens. + */ +export function slugifySegment(segment: string): string { + return segment + .normalize('NFD') // Decompose accented chars + .replace(/[\u0300-\u036f]/g, '') // Strip accent marks + .toLowerCase() + .replace(/[^a-z0-9.\s_-]/g, '') // Keep alphanumeric, dots, spaces, underscores, hyphens + .replace(/[\s]+/g, '-') // Spaces → hyphens + .replace(/-+/g, '-') // Collapse multiple hyphens + .replace(/^-|-$/g, ''); // Strip leading/trailing hyphens +} + +/** + * Slugify a file path: strip .md, normalize separators, slugify each segment. * * Examples: - * people/pedro-franceschi.md → people/pedro-franceschi - * daily/2026-04-05.md → daily/2026-04-05 - * notes.md → notes + * Apple Notes/2017-05-03 ohmygreen.md → apple-notes/2017-05-03-ohmygreen + * people/alice-smith.md → people/alice-smith + * notes/v1.0.0.md → notes/v1.0.0 + */ +export function slugifyPath(filePath: string): string { + let path = filePath.replace(/\.md$/i, ''); + path = path.replace(/\\/g, '/'); + path = path.replace(/^\.?\//, ''); + return path.split('/').map(slugifySegment).filter(Boolean).join('/'); +} + +/** + * Convert a repo-relative file path to a GBrain page slug. */ export function pathToSlug(filePath: string, repoPrefix?: string): string { - // Strip .md extension - let slug = filePath.replace(/\.md$/, ''); - // Normalize separators - slug = slug.replace(/\\/g, '/'); - // Strip leading slash - slug = slug.replace(/^\//, ''); - // Add repo prefix for multi-repo setups + let slug = slugifyPath(filePath); if (repoPrefix) slug = `${repoPrefix}/${slug}`; return slug; } diff --git a/test/e2e/mechanical.test.ts b/test/e2e/mechanical.test.ts index d0e940102..64c7155cb 100644 --- a/test/e2e/mechanical.test.ts +++ b/test/e2e/mechanical.test.ts @@ -697,14 +697,14 @@ describeE2E('E2E: Slug with Special Characters', () => { afterAll(teardownDB); test('imports files with spaces in filename', async () => { - const page = await callOp('get_page', { slug: 'apple-notes/2017-05-03 ohmygreen' }) as any; + const page = await callOp('get_page', { slug: 'apple-notes/2017-05-03-ohmygreen' }) as any; expect(page).not.toBeNull(); expect(page.title).toBe('OhMyGreen'); expect(page.type).toBe('company'); }); test('imports files with parens in filename', async () => { - const page = await callOp('get_page', { slug: 'apple-notes/notes (march 2024)' }) as any; + const page = await callOp('get_page', { slug: 'apple-notes/notes-march-2024' }) as any; expect(page).not.toBeNull(); expect(page.title).toBe('March 2024 Notes'); }); @@ -713,7 +713,7 @@ describeE2E('E2E: Slug with Special Characters', () => { const results = await callOp('search', { query: 'OhMyGreen' }) as any[]; expect(results.length).toBeGreaterThanOrEqual(1); const slugs = results.map((r: any) => r.slug); - expect(slugs).toContain('apple-notes/2017-05-03 ohmygreen'); + expect(slugs).toContain('apple-notes/2017-05-03-ohmygreen'); }); test('re-import of special-char files is idempotent', async () => { diff --git a/test/e2e/sync.test.ts b/test/e2e/sync.test.ts index 62299b5c0..fe3b6eb90 100644 --- a/test/e2e/sync.test.ts +++ b/test/e2e/sync.test.ts @@ -331,4 +331,66 @@ describeE2E('E2E: Git-to-DB Sync Pipeline', () => { noEmbed: true, }); }); + + test('files with spaces in names get slugified slugs', async () => { + const { performSync } = await import('../../src/commands/sync.ts'); + const engine = getEngine(); + + // Add a file with spaces (Apple Notes style) + mkdirSync(join(repoPath, 'Apple Notes'), { recursive: true }); + writeFileSync(join(repoPath, 'Apple Notes/2017-05-03 ohmygreen.md'), [ + '---', + 'title: Ohmygreen Notes', + '---', + '', + 'Notes about ohmygreen lunch service.', + ].join('\n')); + gitCommit(repoPath, 'add apple notes file with spaces'); + + const result = await performSync(engine, { + repoPath, + noPull: true, + noEmbed: true, + }); + + expect(result.status).toBe('synced'); + expect(result.added).toBe(1); + + // Slug should be slugified (lowercase, spaces → hyphens) + const page = await engine.getPage('apple-notes/2017-05-03-ohmygreen'); + expect(page).not.toBeNull(); + expect(page!.title).toBe('Ohmygreen Notes'); + + // Original space-based slug should NOT exist + const rawSlug = await engine.getPage('Apple Notes/2017-05-03 ohmygreen'); + expect(rawSlug).toBeNull(); + }); + + test('incremental sync adds file with special characters', async () => { + const { performSync } = await import('../../src/commands/sync.ts'); + const engine = getEngine(); + + // Add a file with parens and special chars + writeFileSync(join(repoPath, 'Apple Notes/meeting notes (draft).md'), [ + '---', + 'title: Draft Meeting Notes', + '---', + '', + 'Some draft notes from the meeting.', + ].join('\n')); + gitCommit(repoPath, 'add file with parens'); + + const result = await performSync(engine, { + repoPath, + noPull: true, + noEmbed: true, + }); + + expect(result.status).toBe('synced'); + + // Slug should have parens stripped, spaces → hyphens + const page = await engine.getPage('apple-notes/meeting-notes-draft'); + expect(page).not.toBeNull(); + expect(page!.title).toBe('Draft Meeting Notes'); + }); }); diff --git a/test/slug-validation.test.ts b/test/slug-validation.test.ts index 8e69e677f..3761b520d 100644 --- a/test/slug-validation.test.ts +++ b/test/slug-validation.test.ts @@ -1,4 +1,5 @@ import { describe, test, expect } from 'bun:test'; +import { slugifySegment, slugifyPath } from '../src/core/sync.ts'; // Test the validateSlug behavior via the engine // We can't import validateSlug directly (it's private), so we test through putPage mock behavior @@ -10,6 +11,102 @@ function validateSlug(slug: string): boolean { return true; } +describe('slugifySegment', () => { + test('converts spaces to hyphens', () => { + expect(slugifySegment('hello world')).toBe('hello-world'); + }); + + test('strips special characters', () => { + expect(slugifySegment('notes (march 2024)')).toBe('notes-march-2024'); + }); + + test('normalizes unicode accents', () => { + expect(slugifySegment('caf\u00e9')).toBe('cafe'); + }); + + test('collapses multiple hyphens', () => { + expect(slugifySegment('a - b')).toBe('a-b'); + }); + + test('strips leading and trailing hyphens', () => { + expect(slugifySegment(' hello ')).toBe('hello'); + }); + + test('preserves dots', () => { + expect(slugifySegment('v1.0.0')).toBe('v1.0.0'); + }); + + test('preserves underscores', () => { + expect(slugifySegment('my_file_name')).toBe('my_file_name'); + }); + + test('lowercases', () => { + expect(slugifySegment('Apple Notes')).toBe('apple-notes'); + }); + + test('returns empty for all-special-chars input', () => { + expect(slugifySegment('!!!')).toBe(''); + }); + + test('handles curly quotes and ellipsis', () => { + expect(slugifySegment('she\u2026said \u201chello\u201d')).toBe('shesaid-hello'); + }); +}); + +describe('slugifyPath', () => { + test('slugifies each path segment independently', () => { + expect(slugifyPath('Apple Notes/file name.md')).toBe('apple-notes/file-name'); + }); + + test('already-valid slugs unchanged', () => { + expect(slugifyPath('people/alice-smith.md')).toBe('people/alice-smith'); + }); + + test('strips .md extension case-insensitively', () => { + expect(slugifyPath('notes/file.MD')).toBe('notes/file'); + }); + + test('normalizes backslashes', () => { + expect(slugifyPath('notes\\file.md')).toBe('notes/file'); + }); + + test('strips leading ./', () => { + expect(slugifyPath('./notes/file.md')).toBe('notes/file'); + }); + + test('filters empty segments from all-special-chars dirs', () => { + expect(slugifyPath('!!!/file.md')).toBe('file'); + }); + + test('preserves dots in filenames', () => { + expect(slugifyPath('notes/v1.0.0.md')).toBe('notes/v1.0.0'); + }); + + test('handles consecutive slashes', () => { + expect(slugifyPath('a//b.md')).toBe('a/b'); + }); + + // Bug report example transformations + test('Apple Notes example 1', () => { + expect(slugifyPath('Apple Notes/2017-05-03 ohmygreen.md')).toBe('apple-notes/2017-05-03-ohmygreen'); + }); + + test('Apple Notes example 2', () => { + expect(slugifyPath('Apple Notes/2018-12-14 Garry Tan Photo.md')).toBe('apple-notes/2018-12-14-garry-tan-photo'); + }); + + test('Apple Notes example 3 (parens and ellipsis)', () => { + const input = 'Apple Notes/2017-05-05 Today I had a touch base with Kavita for the meeting on Monday. (she\u2026.md'; + const result = slugifyPath(input); + expect(result).toBe('apple-notes/2017-05-05-today-i-had-a-touch-base-with-kavita-for-the-meeting-on-monday.-she'); + }); + + test('meetings transcript example', () => { + expect(slugifyPath('meetings/transcripts/2026-01-21 maria - california c4 collaboration discussion.md')) + .toBe('meetings/transcripts/2026-01-21-maria-california-c4-collaboration-discussion'); + }); +}); + describe('validateSlug (widened for any filename chars)', () => { test('accepts clean slug', () => { expect(validateSlug('people/sarah-chen')).toBe(true); diff --git a/test/sync.test.ts b/test/sync.test.ts index 375479d5d..a10b3571c 100644 --- a/test/sync.test.ts +++ b/test/sync.test.ts @@ -90,12 +90,12 @@ describe('isSyncable', () => { }); describe('pathToSlug', () => { - test('strips .md extension', () => { + test('strips .md extension and lowercases', () => { expect(pathToSlug('people/pedro-franceschi.md')).toBe('people/pedro-franceschi'); }); - test('preserves case', () => { - expect(pathToSlug('People/Pedro-Franceschi.md')).toBe('People/Pedro-Franceschi'); + test('lowercases uppercase paths', () => { + expect(pathToSlug('People/Pedro-Franceschi.md')).toBe('people/pedro-franceschi'); }); test('strips leading slash', () => { @@ -129,6 +129,14 @@ describe('pathToSlug', () => { test('handles file with only extension', () => { expect(pathToSlug('.md')).toBe(''); }); + + test('slugifies spaces to hyphens', () => { + expect(pathToSlug('Apple Notes/2017-05-03 ohmygreen.md')).toBe('apple-notes/2017-05-03-ohmygreen'); + }); + + test('strips special characters', () => { + expect(pathToSlug('notes/meeting (march 2024).md')).toBe('notes/meeting-march-2024'); + }); }); describe('isSyncable edge cases', () => {