From b3891fa7fc0bec3939c356d4f49b7a49de1969b5 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:15:15 -0700 Subject: [PATCH] fix(chunkers/code): tolerate tiktoken special tokens in estimateTokens (#2453) (#3315) Code legitimately contains tiktoken special-token strings (e.g. CLIP/GPT tokenizers embed the literal <|endoftext|>). The default encode() uses disallowed_special='all' and THROWS on those, crashing reindex-code on valid source files. Re-encode treating them as ordinary text (allowed=[], disallowed=[]); heuristic fallback if even that fails. A token COUNT needs no special-token semantics. Co-authored-by: Jim Tang Co-authored-by: Claude Opus 4.8 (1M context) --- src/core/chunkers/code.ts | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/core/chunkers/code.ts b/src/core/chunkers/code.ts index 5578a290a..8145e98ac 100644 --- a/src/core/chunkers/code.ts +++ b/src/core/chunkers/code.ts @@ -1235,7 +1235,25 @@ export function estimateTokens(text: string): number { tiktokenInitialized = true; } if (tiktokenEncoder) { - return tiktokenEncoder.encode(text).length; + try { + return tiktokenEncoder.encode(text).length; + } catch { + // Code legitimately contains tiktoken special-token strings (e.g. CLIP/GPT + // tokenizers embed the literal "<|endoftext|>"). The default encode() uses + // disallowed_special='all' and THROWS on those, crashing reindex-code on + // valid source files. For a token COUNT we don't need special-token + // semantics: re-encode treating them as ordinary text (never throws), + // heuristic only if even that fails. + try { + return ( + tiktokenEncoder as unknown as { + encode: (s: string, allowed: string[], disallowed: string[]) => Uint32Array; + } + ).encode(text, [], []).length; + } catch { + return Math.max(1, Math.ceil(text.length / 4)); + } + } } return Math.max(1, Math.ceil(text.length / 4)); }