gbrain/src/core/text-safe.ts

/**
 * UTF-16-surrogate-safe text helpers.
 *
 * Shared between:
 *   - `src/core/eval-contradictions/judge.ts` — `truncateUtf8` truncates
 *     contradiction-judge prompt inputs to a per-pair char budget without
 *     splitting emoji / non-BMP CJK / mathematical alphanumerics.
 *   - `src/core/cycle/synthesize.ts` — `safeSplitIndex` is the tier-3
 *     hard-split fallback in the dream-cycle chunker. Preserves the D9
 *     stable-chunk-identity invariant by refusing to orphan a high surrogate.
 *
 * Two consumers, two natural shapes:
 *   - `truncateUtf8(text, maxChars) -> string` returns the sliced text.
 *   - `safeSplitIndex(text, maxChars) -> number` returns the boundary index
 *     without allocating the sliced string (cheaper chunker hot path).
 *
 * Both functions cover the same three surrogate cases. The agent-authored
 * `safeSliceEnd` from PRs #1378-#1382 handled only case 1; the AT-low
 * surrogate case (3) silently bit when a chunk boundary landed one
 * position inside an emoji.
 *
 * UTF-16 surrogate ranges:
 *   high surrogate: U+D800..U+DBFF
 *   low surrogate:  U+DC00..U+DFFF
 */

/**
 * Make a string well-formed UTF-16: replace any unpaired surrogate half (lone
 * high U+D800–U+DBFF / lone low U+DC00–U+DFFF) with U+FFFD, preserving valid
 * pairs untouched. A lone surrogate is rejected by Postgres inside a `::jsonb`
 * cast and aborts the WHOLE batch (#2011 — `extract --stale` died at ~1,550
 * pages because `excerpt()` raw-sliced a window boundary through an emoji); it
 * also crashes some JSON transports (the brainstorm cross-prompt path).
 *
 * Uses the Bun/JSC built-in (ES2024). `isWellFormed()` is the cheap guard that
 * avoids the `toWellFormed()` allocation when the string is already clean — the
 * common case. Prefer this over hand-rolled surrogate regexes: the two-pass
 * lookbehind regex mishandles CONSECUTIVE lone low surrogates
 * (`"\uDE80\uDE80"` → `"�\uDE80"`, still malformed), whereas the built-in
 * handles every case (`→ "��"`).
 */
export function ensureWellFormed(s: string): string {
  return s.isWellFormed() ? s : s.toWellFormed();
}

function isHighSurrogate(code: number): boolean {
  return code >= 0xd800 && code <= 0xdbff;
}

function isLowSurrogate(code: number): boolean {
  return code >= 0xdc00 && code <= 0xdfff;
}

/**
 * UTF-8-safe truncation: cap at maxChars but never split a multi-byte
 * character. Returns the text unchanged if already under the limit.
 *
 * Pattern reused from `src/core/minions/handlers/subagent-audit.ts` which
 * faces the same multi-byte concern.
 */
export function truncateUtf8(text: string, maxChars: number): string {
  if (!text) return '';
  if (text.length <= maxChars) return text;
  return text.slice(0, Math.max(0, safeSplitIndex(text, maxChars)));
}

/**
 * Return the largest safe slice index ≤ maxChars (never orphans a UTF-16
 * surrogate). Used by chunkers that need the index itself, not the
 * truncated string.
 *
 * Three back-up cases (mirrors `truncateUtf8`'s implementation exactly,
 * so the two functions cannot drift):
 *   1. Pair STRADDLES the cut: high at maxChars-1, low at maxChars.
 *      Return maxChars-1 (pair starts the next chunk together).
 *   2. Stray high at maxChars-1 (no paired low). Return maxChars-1.
 *   3. Low at maxChars-1 (we're inside a pair that started at maxChars-2).
 *      Return maxChars-2 (whole pair moves to next chunk).
 *
 * Case 3 is intentionally conservative when the pair is COMPLETE in the
 * kept half (e.g. text="abcd🚀", maxChars=6 returns 4, not 6). The 2-char
 * shortfall is harmless for chunker determinism — same input → same
 * chunks every time — and matches truncateUtf8's well-tested behavior.
 * Fixing the conservative back-up would require diverging the two
 * functions; we deliberately match them.
 */
export function safeSplitIndex(text: string, maxChars: number): number {
  if (maxChars <= 0) return 0;
  if (maxChars >= text.length) return text.length;

  const unitAtEnd = text.charCodeAt(maxChars);
  const unitBefore = text.charCodeAt(maxChars - 1);

  // Case 1: pair straddles the cut.
  if (isHighSurrogate(unitBefore) && isLowSurrogate(unitAtEnd)) {
    return maxChars - 1;
  }
  // Case 2: stray high surrogate.
  if (isHighSurrogate(unitBefore)) {
    return maxChars - 1;
  }
  // Case 3: kept half ends at low surrogate; back up two.
  if (isLowSurrogate(unitBefore)) {
    return Math.max(0, maxChars - 2);
  }
  return maxChars;
}