gbrain/test/slug-unicode-scripts.test.ts

import { describe, test, expect } from 'bun:test';
import { slugifySegment, slugifyPath } from '../src/core/sync.ts';
import { validatePageSlug } from '../src/core/operations.ts';
import { isValidHolder } from '../src/core/takes-fence.ts';

/**
 * #3417 — silent data loss for non-Latin, non-CJK scripts.
 *
 * Pre-fix, slugifySegment stripped every character outside [a-z0-9._-] + CJK,
 * so whole filenames in Hebrew / Arabic / Cyrillic / Greek / Thai collapsed to
 * empty segments. Distinct files then mapped to the SAME slug (their shared
 * directory prefix) and last-writer-wins overwrote each other with `import`
 * reporting 0 errors.
 *
 * Every assertion here is behavioral (input → output), so this file FAILS on
 * pre-fix master and passes with the Unicode-property-escape grammar.
 */

describe('#3417: non-Latin scripts survive slugification', () => {
  // The six script families from the issue, before/after.
  const cases: Array<[string, string, string]> = [
    ['Hebrew', 'notes/רשימת קניות.md', 'notes/רשימת-קניות'],
    ['Arabic', 'notes/قائمة المهام.md', 'notes/قائمة-المهام'],
    ['Cyrillic', 'notes/Список задач.md', 'notes/список-задач'],
    // Greek: tonos marks decompose to U+0301 under NFD and are stripped by the
    // same combining-accent pass that turns café → cafe. Consistent, stable.
    ['Greek', 'notes/Λίστα εργασιών.md', 'notes/λιστα-εργασιων'],
    ['Thai', 'notes/รายการซื้อของ.md', 'notes/รายการซื้อของ'],
    ['Hebrew + digits', 'notes/תוכנית עבודה 2026.md', 'notes/תוכנית-עבודה-2026'],
  ];

  for (const [name, input, expected] of cases) {
    test(`${name}: ${input} → ${expected}`, () => {
      expect(slugifyPath(input)).toBe(expected);
    });
  }

  test('distinct same-directory files no longer collapse onto one slug', () => {
    // Pre-fix ALL of these slugified to "notes" — one page, last writer wins.
    const slugs = [
      slugifyPath('notes/רשימת קניות.md'),
      slugifyPath('notes/قائمة المهام.md'),
      slugifyPath('notes/Список задач.md'),
      slugifyPath('notes/Λίστα εργασιών.md'),
      slugifyPath('notes/รายการซื้อของ.md'),
    ];
    expect(new Set(slugs).size).toBe(slugs.length);
    for (const s of slugs) expect(s).not.toBe('notes');
  });

  test('emitted slugs are ACCEPTED by validatePageSlug (three-grammar coherence)', () => {
    // The trap: fixing only sync.ts makes sync emit slugs put_page rejects.
    for (const [, input] of cases) {
      const slug = slugifyPath(input);
      expect(() => validatePageSlug(slug)).not.toThrow();
    }
  });

  test('takes-fence holder grammar accepts non-Latin slugs', () => {
    expect(isValidHolder('people/גארי-כהן')).toBe(true);
    expect(isValidHolder('companies/شركة-مثال')).toBe(true);
    // Uppercase still rejected (lowercase-canonical contract preserved).
    expect(isValidHolder('people/Garry-Tan')).toBe(false);
  });
});

describe('#3417: normalization — NFD (macOS) and NFC (git/Linux) converge', () => {
  test('Hebrew NFD filename produces the same slug as NFC', () => {
    const nfc = 'notes/רשימת קניות.md'.normalize('NFC');
    const nfd = 'notes/רשימת קניות.md'.normalize('NFD');
    expect(slugifyPath(nfd)).toBe(slugifyPath(nfc));
  });

  test('Vietnamese NFD filename produces the same slug as NFC', () => {
    const nfc = 'notes/người dùng.md'.normalize('NFC');
    const nfd = 'notes/người dùng.md'.normalize('NFD');
    expect(slugifyPath(nfd)).toBe(slugifyPath(nfc));
  });
});

describe('#3417: regressions — existing behavior unchanged', () => {
  test('ASCII kebab-casing, lowercasing, dots, underscores', () => {
    expect(slugifyPath('notes/Shopping List.md')).toBe('notes/shopping-list');
    expect(slugifyPath('notes/v1.0.0.md')).toBe('notes/v1.0.0');
    expect(slugifySegment('my_file_name')).toBe('my_file_name');
    expect(slugifySegment('notes (march 2024)')).toBe('notes-march-2024');
  });

  test('Latin accents still strip (café → cafe)', () => {
    expect(slugifySegment('café résumé')).toBe('cafe-resume');
  });

  test('CJK still preserved', () => {
    expect(slugifyPath('notes/购物清单.md')).toBe('notes/购物清单');
    expect(slugifyPath('inbox/品牌圣经.md')).toBe('inbox/品牌圣经');
    expect(slugifySegment('한글테스트'.normalize('NFD'))).toBe('한글테스트');
  });

  test('all-symbol input still collapses to empty (frontmatter-fallback path intact)', () => {
    expect(slugifySegment('!!!')).toBe('');
    expect(slugifySegment('🎉🎉')).toBe('');
  });

  test('control chars, RTL override, punctuation still stripped', () => {
    expect(slugifySegment('evil‮gnp')).toBe('evilgnp');
    expect(slugifySegment('a\u0000b')).toBe('ab');
  });

  test('validatePageSlug still rejects traversal, backslash, RTL override, uppercase-only weirdness', () => {
    expect(() => validatePageSlug('../etc/passwd')).toThrow();
    expect(() => validatePageSlug('notes\\file')).toThrow();
    expect(() => validatePageSlug('notes/‮evil')).toThrow();
    expect(() => validatePageSlug('notes/a\u0007b')).toThrow();
  });
});