diff --git a/src/core/cjk.ts b/src/core/cjk.ts index 49efab249..e0e5c5f3f 100644 --- a/src/core/cjk.ts +++ b/src/core/cjk.ts @@ -21,12 +21,35 @@ export const CJK_SLUG_CHARS = '一-鿿぀-ゟ゠-ヿ가-힯'; export const CJK_RANGES_REGEX = new RegExp(`[${CJK_SLUG_CHARS}]`); /** - * Page-slug segment grammar (no anchors): alnum-or-CJK lead char, then - * alnum/CJK/hyphen continuation. Single source for validatePageSlug - * (operations.ts), SlugRegistry's SLUG_RE, and the dream-cycle - * SUMMARY_SLUG_RE so every slug validator shares one grammar (#738). + * Slug "word" character class (#3417): every script's letters, not just + * Latin + CJK. Unicode property escapes — REQUIRES the `u` flag on any + * regex composed from this string (without `u`, `\p{Ll}` silently matches + * the literal chars `p`, `L`, `l`, `{`, `}`). + * + * \p{Ll} lowercase letters (a-z, Cyrillic/Greek lowercase, đ, …) + * \p{Lm} modifier letters + * \p{Lo} caseless-script letters (Hebrew, Arabic, Thai, CJK, Devanagari, …) + * \p{M} combining marks that survive the Latin accent-strip pass + * (Hebrew niqqud, Arabic harakat, Thai/Devanagari vowel signs) + * \p{N} numbers (0-9, Arabic-Indic digits, …) + * + * Uppercase (\p{Lu}/\p{Lt}) is deliberately excluded: slugifySegment() + * lowercases before filtering, so validators stay lowercase-canonical. + * + * Distinct from CJK_SLUG_CHARS above, which also drives the + * countCJKAwareWords density heuristic — do NOT merge the two, or slug + * grammar changes silently change chunking behavior. */ -export const PAGE_SLUG_SEG = `[a-z0-9${CJK_SLUG_CHARS}][a-z0-9${CJK_SLUG_CHARS}\\-]*`; +export const SLUG_WORD_CHARS = '\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}\\p{N}'; + +/** + * Page-slug segment grammar (no anchors): word-char lead, then word-char or + * hyphen continuation. Single source for validatePageSlug (operations.ts), + * SlugRegistry's SLUG_RE, and the dream-cycle SUMMARY_SLUG_RE so every slug + * validator shares one grammar (#738). Compose with the `u` flag — see + * SLUG_WORD_CHARS. + */ +export const PAGE_SLUG_SEG = `[${SLUG_WORD_CHARS}][${SLUG_WORD_CHARS}\\-]*`; export const CJK_SENTENCE_DELIMITERS = ['。', '！', '？']; // 。！？ export const CJK_CLAUSE_DELIMITERS = ['；', '：', '，', '、']; // ；：，、 diff --git a/src/core/cycle/synthesize.ts b/src/core/cycle/synthesize.ts index 4f1166cec..f786294a3 100644 --- a/src/core/cycle/synthesize.ts +++ b/src/core/cycle/synthesize.ts @@ -48,8 +48,9 @@ import { safeSplitIndex } from '../text-safe.ts'; import { PAGE_SLUG_SEG } from '../cjk.ts'; // Slug grammar from validatePageSlug — shared via PAGE_SLUG_SEG (#738). -// Used for the orchestrator-written summary index slug. -const SUMMARY_SLUG_RE = new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})*$`); +// Used for the orchestrator-written summary index slug. `u` flag required +// by PAGE_SLUG_SEG's \p{...} classes (#3417). +const SUMMARY_SLUG_RE = new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})*$`, 'u'); // ── Model context budget (D1, D5, D7, D9) ───────────────────────────── diff --git a/src/core/import-file.ts b/src/core/import-file.ts index e44d7ed3e..1d6c0b178 100644 --- a/src/core/import-file.ts +++ b/src/core/import-file.ts @@ -1092,8 +1092,8 @@ export async function importFromFile( chunks: 0, error: `Filename "${relativePath}" produces no usable slug. ` + - `Add a "slug:" to the frontmatter, or rename the file to use ` + - `ASCII / Chinese / Japanese / Korean characters.`, + `Add a "slug:" to the frontmatter, or rename the file to include ` + + `at least one letter or number (any script).`, }; } } else if (parsed.slug !== expectedSlug) { diff --git a/src/core/operations.ts b/src/core/operations.ts index d1ee25df6..70f907013 100644 --- a/src/core/operations.ts +++ b/src/core/operations.ts @@ -162,10 +162,11 @@ export function validatePageSlug(slug: string): void { if (slug.length > 255) { throw new OperationError('invalid_params', 'page_slug exceeds 255 characters'); } - // v0.32.7: CJK ranges (Han / Hiragana / Katakana / Hangul Syllables) allowed - // in segments. ASCII shape rules (lead char, hyphen continuation) preserved. - if (!new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})*$`, 'i').test(slug)) { - throw new OperationError('invalid_params', `Invalid page_slug: ${slug} (allowed: alphanumeric, CJK, hyphens, forward-slash separated segments)`); + // #3417: letters/numbers from any script allowed in segments (u flag required + // for the \p{...} classes in PAGE_SLUG_SEG). Shape rules (lead char, hyphen + // continuation) preserved. + if (!new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})*$`, 'iu').test(slug)) { + throw new OperationError('invalid_params', `Invalid page_slug: ${slug} (allowed: letters/numbers in any script, hyphens, forward-slash separated segments)`); } } diff --git a/src/core/output/slug-registry.ts b/src/core/output/slug-registry.ts index f7010139c..5ead7b20b 100644 --- a/src/core/output/slug-registry.ts +++ b/src/core/output/slug-registry.ts @@ -72,9 +72,10 @@ export class SlugRegistryError extends Error { // SlugRegistry // --------------------------------------------------------------------------- -// Shares the page-slug segment grammar (incl. CJK ranges, #738) with +// Shares the page-slug segment grammar (all scripts, #738/#3417) with // validatePageSlug; keeps this site's dir/name shape (>= 2 segments). -const SLUG_RE = new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})+$`); +// `u` flag required by PAGE_SLUG_SEG's \p{...} classes. +const SLUG_RE = new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})+$`, 'u'); export class SlugRegistry { constructor(private engine: BrainEngine) {} diff --git a/src/core/sync.ts b/src/core/sync.ts index af6ff1ba3..1133bf25b 100644 --- a/src/core/sync.ts +++ b/src/core/sync.ts @@ -11,7 +11,7 @@ * pathToSlug() → convert file paths to page slugs */ -import { CJK_SLUG_CHARS } from './cjk.ts'; +import { SLUG_WORD_CHARS } from './cjk.ts'; // v0.37.7.0 #1169 submodule-detection helpers. Bottom-of-file already // aliases existsSync as `_existsSync` for other purposes; the top-of-file // import keeps the pruneDir helper's deps near its callsite. @@ -396,8 +396,10 @@ export function unsyncableReason(path: string, opts: SyncableOptions = {}): Sync /** * Character class for the lowercase-canonical form of a slug segment after - * slugifySegment() has run. Lowercase letters, digits, dots, underscores, - * hyphens. Exposed so adjacent code (e.g. takes-fence holder validation, + * slugifySegment() has run. Letters/numbers in any script (lowercase where + * the script has case — #3417), dots, underscores, hyphens. Uses \p{...} + * classes, so composed regexes need the `u` flag (this one carries it). + * Exposed so adjacent code (e.g. takes-fence holder validation, * v0.32 EXP-4) can reuse the actual repo slug grammar instead of inventing * a stricter parallel one and emitting false-positive warnings on legitimate * `companies/acme.io` / `people/foo_bar` slugs (codex review #3). @@ -405,15 +407,18 @@ export function unsyncableReason(path: string, opts: SyncableOptions = {}): Sync * Pattern is the inner character class only (no anchors); callers wrap it * in `^...$` or compose it with prefixes like `(?:people|companies)/...`. */ -export const SLUG_SEGMENT_PATTERN = new RegExp(`[a-z0-9._\\-${CJK_SLUG_CHARS}]+`); +export const SLUG_SEGMENT_PATTERN = new RegExp(`[${SLUG_WORD_CHARS}._\\-]+`, 'u'); /** * Slugify a single path segment: lowercase, strip special chars, spaces → hyphens. - * CJK ranges (Han / Hiragana / Katakana / Hangul Syllables) are preserved (v0.32.7). - * NFC re-normalize after the NFD-strip-accents pass so Hangul Jamo recomposes back - * into precomposed syllables that fall inside the whitelist. + * Letters and numbers from EVERY script are preserved (#3417): previously only + * Latin + CJK survived, so Hebrew/Arabic/Cyrillic/Greek/Thai/... filenames + * collapsed to empty segments and distinct files silently merged onto one slug. + * NFC re-normalize after the NFD-strip-accents pass so Hangul Jamo recomposes + * back into precomposed syllables, and so NFD filenames (macOS) and NFC + * filenames (Linux/git) of the same name produce the SAME slug. */ -const SLUGIFY_KEEP_RE = new RegExp(`[^a-z0-9.\\s_\\-${CJK_SLUG_CHARS}]`, 'g'); +const SLUGIFY_KEEP_RE = new RegExp(`[^${SLUG_WORD_CHARS}.\\s_\\-]`, 'gu'); export function slugifySegment(segment: string): string { return segment diff --git a/src/core/takes-fence.ts b/src/core/takes-fence.ts index e74d214e3..2a33e8622 100644 --- a/src/core/takes-fence.ts +++ b/src/core/takes-fence.ts @@ -134,6 +134,7 @@ export const TAKES_FENCE_END = ''; import { SLUG_SEGMENT_PATTERN } from './sync.ts'; export const HOLDER_REGEX = new RegExp( `^(?:world|brain|(?:people|companies)/${SLUG_SEGMENT_PATTERN.source}|${SLUG_SEGMENT_PATTERN.source})$`, + 'u', // required by SLUG_SEGMENT_PATTERN's \p{...} classes (#3417) ); /** diff --git a/test/import-file.test.ts b/test/import-file.test.ts index 86b306c49..820c7585e 100644 --- a/test/import-file.test.ts +++ b/test/import-file.test.ts @@ -522,7 +522,7 @@ just content. const result = await importFile(engine, filePath, '🌟🚀.md', { noEmbed: true }); expect(result.status).toBe('skipped'); expect(result.error).toContain('no usable slug'); - expect(result.error).toContain('ASCII / Chinese / Japanese / Korean'); + expect(result.error).toContain('at least one letter or number (any script)'); expect((engine as any)._calls.length).toBe(0); }); diff --git a/test/slug-unicode-scripts.test.ts b/test/slug-unicode-scripts.test.ts new file mode 100644 index 000000000..71a95671a --- /dev/null +++ b/test/slug-unicode-scripts.test.ts @@ -0,0 +1,115 @@ +import { describe, test, expect } from 'bun:test'; +import { slugifySegment, slugifyPath } from '../src/core/sync.ts'; +import { validatePageSlug } from '../src/core/operations.ts'; +import { isValidHolder } from '../src/core/takes-fence.ts'; + +/** + * #3417 — silent data loss for non-Latin, non-CJK scripts. + * + * Pre-fix, slugifySegment stripped every character outside [a-z0-9._-] + CJK, + * so whole filenames in Hebrew / Arabic / Cyrillic / Greek / Thai collapsed to + * empty segments. Distinct files then mapped to the SAME slug (their shared + * directory prefix) and last-writer-wins overwrote each other with `import` + * reporting 0 errors. + * + * Every assertion here is behavioral (input → output), so this file FAILS on + * pre-fix master and passes with the Unicode-property-escape grammar. + */ + +describe('#3417: non-Latin scripts survive slugification', () => { + // The six script families from the issue, before/after. + const cases: Array<[string, string, string]> = [ + ['Hebrew', 'notes/רשימת קניות.md', 'notes/רשימת-קניות'], + ['Arabic', 'notes/قائمة المهام.md', 'notes/قائمة-المهام'], + ['Cyrillic', 'notes/Список задач.md', 'notes/список-задач'], + // Greek: tonos marks decompose to U+0301 under NFD and are stripped by the + // same combining-accent pass that turns café → cafe. Consistent, stable. + ['Greek', 'notes/Λίστα εργασιών.md', 'notes/λιστα-εργασιων'], + ['Thai', 'notes/รายการซื้อของ.md', 'notes/รายการซื้อของ'], + ['Hebrew + digits', 'notes/תוכנית עבודה 2026.md', 'notes/תוכנית-עבודה-2026'], + ]; + + for (const [name, input, expected] of cases) { + test(`${name}: ${input} → ${expected}`, () => { + expect(slugifyPath(input)).toBe(expected); + }); + } + + test('distinct same-directory files no longer collapse onto one slug', () => { + // Pre-fix ALL of these slugified to "notes" — one page, last writer wins. + const slugs = [ + slugifyPath('notes/רשימת קניות.md'), + slugifyPath('notes/قائمة المهام.md'), + slugifyPath('notes/Список задач.md'), + slugifyPath('notes/Λίστα εργασιών.md'), + slugifyPath('notes/รายการซื้อของ.md'), + ]; + expect(new Set(slugs).size).toBe(slugs.length); + for (const s of slugs) expect(s).not.toBe('notes'); + }); + + test('emitted slugs are ACCEPTED by validatePageSlug (three-grammar coherence)', () => { + // The trap: fixing only sync.ts makes sync emit slugs put_page rejects. + for (const [, input] of cases) { + const slug = slugifyPath(input); + expect(() => validatePageSlug(slug)).not.toThrow(); + } + }); + + test('takes-fence holder grammar accepts non-Latin slugs', () => { + expect(isValidHolder('people/גארי-כהן')).toBe(true); + expect(isValidHolder('companies/شركة-مثال')).toBe(true); + // Uppercase still rejected (lowercase-canonical contract preserved). + expect(isValidHolder('people/Garry-Tan')).toBe(false); + }); +}); + +describe('#3417: normalization — NFD (macOS) and NFC (git/Linux) converge', () => { + test('Hebrew NFD filename produces the same slug as NFC', () => { + const nfc = 'notes/רשימת קניות.md'.normalize('NFC'); + const nfd = 'notes/רשימת קניות.md'.normalize('NFD'); + expect(slugifyPath(nfd)).toBe(slugifyPath(nfc)); + }); + + test('Vietnamese NFD filename produces the same slug as NFC', () => { + const nfc = 'notes/người dùng.md'.normalize('NFC'); + const nfd = 'notes/người dùng.md'.normalize('NFD'); + expect(slugifyPath(nfd)).toBe(slugifyPath(nfc)); + }); +}); + +describe('#3417: regressions — existing behavior unchanged', () => { + test('ASCII kebab-casing, lowercasing, dots, underscores', () => { + expect(slugifyPath('notes/Shopping List.md')).toBe('notes/shopping-list'); + expect(slugifyPath('notes/v1.0.0.md')).toBe('notes/v1.0.0'); + expect(slugifySegment('my_file_name')).toBe('my_file_name'); + expect(slugifySegment('notes (march 2024)')).toBe('notes-march-2024'); + }); + + test('Latin accents still strip (café → cafe)', () => { + expect(slugifySegment('café résumé')).toBe('cafe-resume'); + }); + + test('CJK still preserved', () => { + expect(slugifyPath('notes/购物清单.md')).toBe('notes/购物清单'); + expect(slugifyPath('inbox/品牌圣经.md')).toBe('inbox/品牌圣经'); + expect(slugifySegment('한글테스트'.normalize('NFD'))).toBe('한글테스트'); + }); + + test('all-symbol input still collapses to empty (frontmatter-fallback path intact)', () => { + expect(slugifySegment('!!!')).toBe(''); + expect(slugifySegment('🎉🎉')).toBe(''); + }); + + test('control chars, RTL override, punctuation still stripped', () => { + expect(slugifySegment('evil‮gnp')).toBe('evilgnp'); + expect(slugifySegment('a\u0000b')).toBe('ab'); + }); + + test('validatePageSlug still rejects traversal, backslash, RTL override, uppercase-only weirdness', () => { + expect(() => validatePageSlug('../etc/passwd')).toThrow(); + expect(() => validatePageSlug('notes\\file')).toThrow(); + expect(() => validatePageSlug('notes/‮evil')).toThrow(); + expect(() => validatePageSlug('notes/a\u0007b')).toThrow(); + }); +}); diff --git a/test/slug-validation.test.ts b/test/slug-validation.test.ts index 7100d2b2f..ed54eb9e1 100644 --- a/test/slug-validation.test.ts +++ b/test/slug-validation.test.ts @@ -259,10 +259,10 @@ describe('SLUG_SEGMENT_PATTERN (v0.32.7)', () => { expect(SLUG_SEGMENT_PATTERN.test('icp-理想客户画像')).toBe(true); }); - test('REGRESSION: rejects non-CJK Unicode (Vietnamese)', () => { - // Scope is CJK only; Vietnamese with combining diacritics stays rejected - // until we widen to Unicode property escapes in v0.33+. - const result = 'người-dùng'.match(new RegExp(`^${SLUG_SEGMENT_PATTERN.source}$`)); - expect(result).toBeNull(); + test('accepts non-CJK Unicode (Vietnamese) since the #3417 all-script widening', () => { + // Pre-#3417 this was rejected (scope was CJK only). The grammar now uses + // Unicode property escapes, so đ/ư/etc. are valid slug characters. + const result = 'người-dùng'.match(new RegExp(`^${SLUG_SEGMENT_PATTERN.source}$`, 'u')); + expect(result).not.toBeNull(); }); });