From 784358f5fddd67f24cdc51359b4c28092a5056fa Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:11:41 -0700 Subject: [PATCH] fix(sync): preserve non-Latin scripts in slugs (#3417) (#3522) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit slugifySegment stripped every character outside [a-z0-9._-] + four CJK ranges, so filenames in Hebrew, Arabic, Cyrillic, Greek, Thai (and every other script) collapsed to empty segments. Distinct files then mapped to the SAME slug (their shared directory prefix) and silently overwrote each other, last-writer-wins, with import reporting 0 errors. All three slug grammars move together so sync never emits a slug that put_page rejects: - sync.ts SLUGIFY_KEEP_RE + SLUG_SEGMENT_PATTERN — now keep \p{Ll}\p{Lm}\p{Lo}\p{M}\p{N} (new single-source SLUG_WORD_CHARS in cjk.ts), with the u flag - cjk.ts PAGE_SLUG_SEG — rebuilt on SLUG_WORD_CHARS; consumers (validatePageSlug, SlugRegistry SLUG_RE, dream-cycle SUMMARY_SLUG_RE, takes-fence HOLDER_REGEX) all gain the required u flag CJK_SLUG_CHARS is untouched — it also drives the countCJKAwareWords chunking density heuristic and must not move with slug grammar. Kebab-casing, lowercasing, Latin accent-strip (cafe), dots/underscores, path handling, and the NFC re-normalize (NFD macOS filenames converge with NFC git filenames) are all unchanged and regression-pinned in test/slug-unicode-scripts.test.ts. No data migration: the collapse was N-to-1, so only one row per group ever persisted; a normal gbrain sync recreates the lost pages under their real slugs. Co-authored-by: Garry Tan Co-authored-by: Claude Opus 5 --- src/core/cjk.ts | 33 +++++++-- src/core/cycle/synthesize.ts | 5 +- src/core/import-file.ts | 4 +- src/core/operations.ts | 9 +-- src/core/output/slug-registry.ts | 5 +- src/core/sync.ts | 21 +++--- src/core/takes-fence.ts | 1 + test/import-file.test.ts | 2 +- test/slug-unicode-scripts.test.ts | 115 ++++++++++++++++++++++++++++++ test/slug-validation.test.ts | 10 +-- 10 files changed, 176 insertions(+), 29 deletions(-) create mode 100644 test/slug-unicode-scripts.test.ts diff --git a/src/core/cjk.ts b/src/core/cjk.ts index 49efab249..e0e5c5f3f 100644 --- a/src/core/cjk.ts +++ b/src/core/cjk.ts @@ -21,12 +21,35 @@ export const CJK_SLUG_CHARS = '一-鿿぀-ゟ゠-ヿ가-힯'; export const CJK_RANGES_REGEX = new RegExp(`[${CJK_SLUG_CHARS}]`); /** - * Page-slug segment grammar (no anchors): alnum-or-CJK lead char, then - * alnum/CJK/hyphen continuation. Single source for validatePageSlug - * (operations.ts), SlugRegistry's SLUG_RE, and the dream-cycle - * SUMMARY_SLUG_RE so every slug validator shares one grammar (#738). + * Slug "word" character class (#3417): every script's letters, not just + * Latin + CJK. Unicode property escapes — REQUIRES the `u` flag on any + * regex composed from this string (without `u`, `\p{Ll}` silently matches + * the literal chars `p`, `L`, `l`, `{`, `}`). + * + * \p{Ll} lowercase letters (a-z, Cyrillic/Greek lowercase, đ, …) + * \p{Lm} modifier letters + * \p{Lo} caseless-script letters (Hebrew, Arabic, Thai, CJK, Devanagari, …) + * \p{M} combining marks that survive the Latin accent-strip pass + * (Hebrew niqqud, Arabic harakat, Thai/Devanagari vowel signs) + * \p{N} numbers (0-9, Arabic-Indic digits, …) + * + * Uppercase (\p{Lu}/\p{Lt}) is deliberately excluded: slugifySegment() + * lowercases before filtering, so validators stay lowercase-canonical. + * + * Distinct from CJK_SLUG_CHARS above, which also drives the + * countCJKAwareWords density heuristic — do NOT merge the two, or slug + * grammar changes silently change chunking behavior. */ -export const PAGE_SLUG_SEG = `[a-z0-9${CJK_SLUG_CHARS}][a-z0-9${CJK_SLUG_CHARS}\\-]*`; +export const SLUG_WORD_CHARS = '\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}\\p{N}'; + +/** + * Page-slug segment grammar (no anchors): word-char lead, then word-char or + * hyphen continuation. Single source for validatePageSlug (operations.ts), + * SlugRegistry's SLUG_RE, and the dream-cycle SUMMARY_SLUG_RE so every slug + * validator shares one grammar (#738). Compose with the `u` flag — see + * SLUG_WORD_CHARS. + */ +export const PAGE_SLUG_SEG = `[${SLUG_WORD_CHARS}][${SLUG_WORD_CHARS}\\-]*`; export const CJK_SENTENCE_DELIMITERS = ['。', '！', '？']; // 。！？ export const CJK_CLAUSE_DELIMITERS = ['；', '：', '，', '、']; // ；：，、 diff --git a/src/core/cycle/synthesize.ts b/src/core/cycle/synthesize.ts index 4f1166cec..f786294a3 100644 --- a/src/core/cycle/synthesize.ts +++ b/src/core/cycle/synthesize.ts @@ -48,8 +48,9 @@ import { safeSplitIndex } from '../text-safe.ts'; import { PAGE_SLUG_SEG } from '../cjk.ts'; // Slug grammar from validatePageSlug — shared via PAGE_SLUG_SEG (#738). -// Used for the orchestrator-written summary index slug. -const SUMMARY_SLUG_RE = new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})*$`); +// Used for the orchestrator-written summary index slug. `u` flag required +// by PAGE_SLUG_SEG's \p{...} classes (#3417). +const SUMMARY_SLUG_RE = new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})*$`, 'u'); // ── Model context budget (D1, D5, D7, D9) ───────────────────────────── diff --git a/src/core/import-file.ts b/src/core/import-file.ts index e44d7ed3e..1d6c0b178 100644 --- a/src/core/import-file.ts +++ b/src/core/import-file.ts @@ -1092,8 +1092,8 @@ export async function importFromFile( chunks: 0, error: `Filename "${relativePath}" produces no usable slug. ` + - `Add a "slug:" to the frontmatter, or rename the file to use ` + - `ASCII / Chinese / Japanese / Korean characters.`, + `Add a "slug:" to the frontmatter, or rename the file to include ` + + `at least one letter or number (any script).`, }; } } else if (parsed.slug !== expectedSlug) { diff --git a/src/core/operations.ts b/src/core/operations.ts index d1ee25df6..70f907013 100644 --- a/src/core/operations.ts +++ b/src/core/operations.ts @@ -162,10 +162,11 @@ export function validatePageSlug(slug: string): void { if (slug.length > 255) { throw new OperationError('invalid_params', 'page_slug exceeds 255 characters'); } - // v0.32.7: CJK ranges (Han / Hiragana / Katakana / Hangul Syllables) allowed - // in segments. ASCII shape rules (lead char, hyphen continuation) preserved. - if (!new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})*$`, 'i').test(slug)) { - throw new OperationError('invalid_params', `Invalid page_slug: ${slug} (allowed: alphanumeric, CJK, hyphens, forward-slash separated segments)`); + // #3417: letters/numbers from any script allowed in segments (u flag required + // for the \p{...} classes in PAGE_SLUG_SEG). Shape rules (lead char, hyphen + // continuation) preserved. + if (!new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})*$`, 'iu').test(slug)) { + throw new OperationError('invalid_params', `Invalid page_slug: ${slug} (allowed: letters/numbers in any script, hyphens, forward-slash separated segments)`); } } diff --git a/src/core/output/slug-registry.ts b/src/core/output/slug-registry.ts index f7010139c..5ead7b20b 100644 --- a/src/core/output/slug-registry.ts +++ b/src/core/output/slug-registry.ts @@ -72,9 +72,10 @@ export class SlugRegistryError extends Error { // SlugRegistry // --------------------------------------------------------------------------- -// Shares the page-slug segment grammar (incl. CJK ranges, #738) with +// Shares the page-slug segment grammar (all scripts, #738/#3417) with // validatePageSlug; keeps this site's dir/name shape (>= 2 segments). -const SLUG_RE = new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})+$`); +// `u` flag required by PAGE_SLUG_SEG's \p{...} classes. +const SLUG_RE = new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})+$`, 'u'); export class SlugRegistry { constructor(private engine: BrainEngine) {} diff --git a/src/core/sync.ts b/src/core/sync.ts index af6ff1ba3..1133bf25b 100644 --- a/src/core/sync.ts +++ b/src/core/sync.ts @@ -11,7 +11,7 @@ * pathToSlug() → convert file paths to page slugs */ -import { CJK_SLUG_CHARS } from './cjk.ts'; +import { SLUG_WORD_CHARS } from './cjk.ts'; // v0.37.7.0 #1169 submodule-detection helpers. Bottom-of-file already // aliases existsSync as `_existsSync` for other purposes; the top-of-file // import keeps the pruneDir helper's deps near its callsite. @@ -396,8 +396,10 @@ export function unsyncableReason(path: string, opts: SyncableOptions = {}): Sync /** * Character class for the lowercase-canonical form of a slug segment after - * slugifySegment() has run. Lowercase letters, digits, dots, underscores, - * hyphens. Exposed so adjacent code (e.g. takes-fence holder validation, + * slugifySegment() has run. Letters/numbers in any script (lowercase where + * the script has case — #3417), dots, underscores, hyphens. Uses \p{...} + * classes, so composed regexes need the `u` flag (this one carries it). + * Exposed so adjacent code (e.g. takes-fence holder validation, * v0.32 EXP-4) can reuse the actual repo slug grammar instead of inventing * a stricter parallel one and emitting false-positive warnings on legitimate * `companies/acme.io` / `people/foo_bar` slugs (codex review #3). @@ -405,15 +407,18 @@ export function unsyncableReason(path: string, opts: SyncableOptions = {}): Sync * Pattern is the inner character class only (no anchors); callers wrap it * in `^...$` or compose it with prefixes like `(?:people|companies)/...`. */ -export const SLUG_SEGMENT_PATTERN = new RegExp(`[a-z0-9._\\-${CJK_SLUG_CHARS}]+`); +export const SLUG_SEGMENT_PATTERN = new RegExp(`[${SLUG_WORD_CHARS}._\\-]+`, 'u'); /** * Slugify a single path segment: lowercase, strip special chars, spaces → hyphens. - * CJK ranges (Han / Hiragana / Katakana / Hangul Syllables) are preserved (v0.32.7). - * NFC re-normalize after the NFD-strip-accents pass so Hangul Jamo recomposes back - * into precomposed syllables that fall inside the whitelist. + * Letters and numbers from EVERY script are preserved (#3417): previously only + * Latin + CJK survived, so Hebrew/Arabic/Cyrillic/Greek/Thai/... filenames + * collapsed to empty segments and distinct files silently merged onto one slug. + * NFC re-normalize after the NFD-strip-accents pass so Hangul Jamo recomposes + * back into precomposed syllables, and so NFD filenames (macOS) and NFC + * filenames (Linux/git) of the same name produce the SAME slug. */ -const SLUGIFY_KEEP_RE = new RegExp(`[^a-z0-9.\\s_\\-${CJK_SLUG_CHARS}]`, 'g'); +const SLUGIFY_KEEP_RE = new RegExp(`[^${SLUG_WORD_CHARS}.\\s_\\-]`, 'gu'); export function slugifySegment(segment: string): string { return segment diff --git a/src/core/takes-fence.ts b/src/core/takes-fence.ts index e74d214e3..2a33e8622 100644 --- a/src/core/takes-fence.ts +++ b/src/core/takes-fence.ts @@ -134,6 +134,7 @@ export const TAKES_FENCE_END = ''; import { SLUG_SEGMENT_PATTERN } from './sync.ts'; export const HOLDER_REGEX = new RegExp( `^(?:world|brain|(?:people|companies)/${SLUG_SEGMENT_PATTERN.source}|${SLUG_SEGMENT_PATTERN.source})$`, + 'u', // required by SLUG_SEGMENT_PATTERN's \p{...} classes (#3417) ); /** diff --git a/test/import-file.test.ts b/test/import-file.test.ts index 86b306c49..820c7585e 100644 --- a/test/import-file.test.ts +++ b/test/import-file.test.ts @@ -522,7 +522,7 @@ just content. const result = await importFile(engine, filePath, '🌟🚀.md', { noEmbed: true }); expect(result.status).toBe('skipped'); expect(result.error).toContain('no usable slug'); - expect(result.error).toContain('ASCII / Chinese / Japanese / Korean'); + expect(result.error).toContain('at least one letter or number (any script)'); expect((engine as any)._calls.length).toBe(0); }); diff --git a/test/slug-unicode-scripts.test.ts b/test/slug-unicode-scripts.test.ts new file mode 100644 index 000000000..71a95671a --- /dev/null +++ b/test/slug-unicode-scripts.test.ts @@ -0,0 +1,115 @@ +import { describe, test, expect } from 'bun:test'; +import { slugifySegment, slugifyPath } from '../src/core/sync.ts'; +import { validatePageSlug } from '../src/core/operations.ts'; +import { isValidHolder } from '../src/core/takes-fence.ts'; + +/** + * #3417 — silent data loss for non-Latin, non-CJK scripts. + * + * Pre-fix, slugifySegment stripped every character outside [a-z0-9._-] + CJK, + * so whole filenames in Hebrew / Arabic / Cyrillic / Greek / Thai collapsed to + * empty segments. Distinct files then mapped to the SAME slug (their shared + * directory prefix) and last-writer-wins overwrote each other with `import` + * reporting 0 errors. + * + * Every assertion here is behavioral (input → output), so this file FAILS on + * pre-fix master and passes with the Unicode-property-escape grammar. + */ + +describe('#3417: non-Latin scripts survive slugification', () => { + // The six script families from the issue, before/after. + const cases: Array<[string, string, string]> = [ + ['Hebrew', 'notes/רשימת קניות.md', 'notes/רשימת-קניות'], + ['Arabic', 'notes/قائمة المهام.md', 'notes/قائمة-المهام'], + ['Cyrillic', 'notes/Список задач.md', 'notes/список-задач'], + // Greek: tonos marks decompose to U+0301 under NFD and are stripped by the + // same combining-accent pass that turns café → cafe. Consistent, stable. + ['Greek', 'notes/Λίστα εργασιών.md', 'notes/λιστα-εργασιων'], + ['Thai', 'notes/รายการซื้อของ.md', 'notes/รายการซื้อของ'], + ['Hebrew + digits', 'notes/תוכנית עבודה 2026.md', 'notes/תוכנית-עבודה-2026'], + ]; + + for (const [name, input, expected] of cases) { + test(`${name}: ${input} → ${expected}`, () => { + expect(slugifyPath(input)).toBe(expected); + }); + } + + test('distinct same-directory files no longer collapse onto one slug', () => { + // Pre-fix ALL of these slugified to "notes" — one page, last writer wins. + const slugs = [ + slugifyPath('notes/רשימת קניות.md'), + slugifyPath('notes/قائمة المهام.md'), + slugifyPath('notes/Список задач.md'), + slugifyPath('notes/Λίστα εργασιών.md'), + slugifyPath('notes/รายการซื้อของ.md'), + ]; + expect(new Set(slugs).size).toBe(slugs.length); + for (const s of slugs) expect(s).not.toBe('notes'); + }); + + test('emitted slugs are ACCEPTED by validatePageSlug (three-grammar coherence)', () => { + // The trap: fixing only sync.ts makes sync emit slugs put_page rejects. + for (const [, input] of cases) { + const slug = slugifyPath(input); + expect(() => validatePageSlug(slug)).not.toThrow(); + } + }); + + test('takes-fence holder grammar accepts non-Latin slugs', () => { + expect(isValidHolder('people/גארי-כהן')).toBe(true); + expect(isValidHolder('companies/شركة-مثال')).toBe(true); + // Uppercase still rejected (lowercase-canonical contract preserved). + expect(isValidHolder('people/Garry-Tan')).toBe(false); + }); +}); + +describe('#3417: normalization — NFD (macOS) and NFC (git/Linux) converge', () => { + test('Hebrew NFD filename produces the same slug as NFC', () => { + const nfc = 'notes/רשימת קניות.md'.normalize('NFC'); + const nfd = 'notes/רשימת קניות.md'.normalize('NFD'); + expect(slugifyPath(nfd)).toBe(slugifyPath(nfc)); + }); + + test('Vietnamese NFD filename produces the same slug as NFC', () => { + const nfc = 'notes/người dùng.md'.normalize('NFC'); + const nfd = 'notes/người dùng.md'.normalize('NFD'); + expect(slugifyPath(nfd)).toBe(slugifyPath(nfc)); + }); +}); + +describe('#3417: regressions — existing behavior unchanged', () => { + test('ASCII kebab-casing, lowercasing, dots, underscores', () => { + expect(slugifyPath('notes/Shopping List.md')).toBe('notes/shopping-list'); + expect(slugifyPath('notes/v1.0.0.md')).toBe('notes/v1.0.0'); + expect(slugifySegment('my_file_name')).toBe('my_file_name'); + expect(slugifySegment('notes (march 2024)')).toBe('notes-march-2024'); + }); + + test('Latin accents still strip (café → cafe)', () => { + expect(slugifySegment('café résumé')).toBe('cafe-resume'); + }); + + test('CJK still preserved', () => { + expect(slugifyPath('notes/购物清单.md')).toBe('notes/购物清单'); + expect(slugifyPath('inbox/品牌圣经.md')).toBe('inbox/品牌圣经'); + expect(slugifySegment('한글테스트'.normalize('NFD'))).toBe('한글테스트'); + }); + + test('all-symbol input still collapses to empty (frontmatter-fallback path intact)', () => { + expect(slugifySegment('!!!')).toBe(''); + expect(slugifySegment('🎉🎉')).toBe(''); + }); + + test('control chars, RTL override, punctuation still stripped', () => { + expect(slugifySegment('evil‮gnp')).toBe('evilgnp'); + expect(slugifySegment('a\u0000b')).toBe('ab'); + }); + + test('validatePageSlug still rejects traversal, backslash, RTL override, uppercase-only weirdness', () => { + expect(() => validatePageSlug('../etc/passwd')).toThrow(); + expect(() => validatePageSlug('notes\\file')).toThrow(); + expect(() => validatePageSlug('notes/‮evil')).toThrow(); + expect(() => validatePageSlug('notes/a\u0007b')).toThrow(); + }); +}); diff --git a/test/slug-validation.test.ts b/test/slug-validation.test.ts index 7100d2b2f..ed54eb9e1 100644 --- a/test/slug-validation.test.ts +++ b/test/slug-validation.test.ts @@ -259,10 +259,10 @@ describe('SLUG_SEGMENT_PATTERN (v0.32.7)', () => { expect(SLUG_SEGMENT_PATTERN.test('icp-理想客户画像')).toBe(true); }); - test('REGRESSION: rejects non-CJK Unicode (Vietnamese)', () => { - // Scope is CJK only; Vietnamese with combining diacritics stays rejected - // until we widen to Unicode property escapes in v0.33+. - const result = 'người-dùng'.match(new RegExp(`^${SLUG_SEGMENT_PATTERN.source}$`)); - expect(result).toBeNull(); + test('accepts non-CJK Unicode (Vietnamese) since the #3417 all-script widening', () => { + // Pre-#3417 this was rejected (scope was CJK only). The grammar now uses + // Unicode property escapes, so đ/ư/etc. are valid slug characters. + const result = 'người-dùng'.match(new RegExp(`^${SLUG_SEGMENT_PATTERN.source}$`, 'u')); + expect(result).not.toBeNull(); }); });