admin/gbrain
1
0
Fork 0
mirror of https://github.com/garrytan/gbrain.git synced 2026-07-27 22:15:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revert "fix(chunkers/code): tolerate tiktoken special tokens in estimateTokens (#2453)"

Browse Source 
This reverts commit b7f70970c1.
This commit is contained in:
Garry Tan
2026-07-23 09:16:17 -07:00
parent a6aafddd23
commit 94535fc0e0
1 changed files with 1 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/chunkers/code.ts
+1 -19
View File
@@ -1235,25 +1235,7 @@ export function estimateTokens(text: string): number {
tiktokenInitialized = true;
}
if (tiktokenEncoder) {
try {
return tiktokenEncoder.encode(text).length;
} catch {
// Code legitimately contains tiktoken special-token strings (e.g. CLIP/GPT
// tokenizers embed the literal "<|endoftext|>"). The default encode() uses
// disallowed_special='all' and THROWS on those, crashing reindex-code on
// valid source files. For a token COUNT we don't need special-token
// semantics: re-encode treating them as ordinary text (never throws),
// heuristic only if even that fails.
try {
return (
tiktokenEncoder as unknown as {
encode: (s: string, allowed: string[], disallowed: string[]) => Uint32Array;
}
).encode(text, [], []).length;
} catch {
return Math.max(1, Math.ceil(text.length / 4));
}
}
return tiktokenEncoder.encode(text).length;
}
return Math.max(1, Math.ceil(text.length / 4));
}