From 94535fc0e0f00cc6c8e5b8623b719808970bfee9 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Thu, 23 Jul 2026 09:16:17 -0700 Subject: [PATCH] Revert "fix(chunkers/code): tolerate tiktoken special tokens in estimateTokens (#2453)" This reverts commit b7f70970c1d95fb6dd202dc9d4a8b34258c309fc. --- src/core/chunkers/code.ts | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/src/core/chunkers/code.ts b/src/core/chunkers/code.ts index 8145e98ac..5578a290a 100644 --- a/src/core/chunkers/code.ts +++ b/src/core/chunkers/code.ts @@ -1235,25 +1235,7 @@ export function estimateTokens(text: string): number { tiktokenInitialized = true; } if (tiktokenEncoder) { - try { - return tiktokenEncoder.encode(text).length; - } catch { - // Code legitimately contains tiktoken special-token strings (e.g. CLIP/GPT - // tokenizers embed the literal "<|endoftext|>"). The default encode() uses - // disallowed_special='all' and THROWS on those, crashing reindex-code on - // valid source files. For a token COUNT we don't need special-token - // semantics: re-encode treating them as ordinary text (never throws), - // heuristic only if even that fails. - try { - return ( - tiktokenEncoder as unknown as { - encode: (s: string, allowed: string[], disallowed: string[]) => Uint32Array; - } - ).encode(text, [], []).length; - } catch { - return Math.max(1, Math.ceil(text.length / 4)); - } - } + return tiktokenEncoder.encode(text).length; } return Math.max(1, Math.ceil(text.length / 4)); }