From b7f70970c1d95fb6dd202dc9d4a8b34258c309fc Mon Sep 17 00:00:00 2001 From: Jim Tang Date: Thu, 23 Jul 2026 08:12:08 -0400 Subject: [PATCH] fix(chunkers/code): tolerate tiktoken special tokens in estimateTokens (#2453) Code legitimately contains tiktoken special-token strings (e.g. CLIP/GPT tokenizers embed the literal <|endoftext|>). The default encode() uses disallowed_special='all' and THROWS on those, crashing reindex-code on valid source files. Re-encode treating them as ordinary text (allowed=[], disallowed=[]); heuristic fallback if even that fails. A token COUNT needs no special-token semantics. Co-authored-by: Claude Opus 4.8 (1M context) --- src/core/chunkers/code.ts | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/core/chunkers/code.ts b/src/core/chunkers/code.ts index 5578a290a..8145e98ac 100644 --- a/src/core/chunkers/code.ts +++ b/src/core/chunkers/code.ts @@ -1235,7 +1235,25 @@ export function estimateTokens(text: string): number { tiktokenInitialized = true; } if (tiktokenEncoder) { - return tiktokenEncoder.encode(text).length; + try { + return tiktokenEncoder.encode(text).length; + } catch { + // Code legitimately contains tiktoken special-token strings (e.g. CLIP/GPT + // tokenizers embed the literal "<|endoftext|>"). The default encode() uses + // disallowed_special='all' and THROWS on those, crashing reindex-code on + // valid source files. For a token COUNT we don't need special-token + // semantics: re-encode treating them as ordinary text (never throws), + // heuristic only if even that fails. + try { + return ( + tiktokenEncoder as unknown as { + encode: (s: string, allowed: string[], disallowed: string[]) => Uint32Array; + } + ).encode(text, [], []).length; + } catch { + return Math.max(1, Math.ceil(text.length / 4)); + } + } } return Math.max(1, Math.ceil(text.length / 4)); }