Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🧢 fix: Bound Tokenizer Work Per Input (#15246)
Danny Avila6d8b1cb01356
diff --git a/packages/api/src/utils/tokenizer.spec.ts b/packages/api/src/utils/tokenizer.spec.tsindex d2a4ccb6a5..1b55b31ca6 100644--- a/packages/api/src/utils/tokenizer.spec.ts+++ b/packages/api/src/utils/tokenizer.spec.ts@@ -14,6 +14,12 @@ describe('Tokenizer', () => {});describe('initEncoding', () => {+ it('uses the established estimate while the encoding initializes', () => {+ const text = 'Cold start token count';++ expect(Tokenizer.getTokenCount(text, 'o200k_base')).toBe(Math.ceil(text.length / 4));+ });+it('should load o200k_base encoding', async () => {await Tokenizer.initEncoding('o200k_base');const count = Tokenizer.getTokenCount('Hello, world!', 'o200k_base');@@ -53,6 +59,20 @@ describe('Tokenizer', () => {const count = Tokenizer.getTokenCount('Hello, world!', 'claude');expect(count).toBeGreaterThan(0);});++ it.each([+ { label: 'an uninterrupted non-whitespace run', text: '_'.repeat(4 * 1024 + 1) },+ { label: 'an oversized input', text: 'word '.repeat(1024) },+ { label: 'multibyte input', text: '界'.repeat(4 * 1024 + 1) },+ ])('uses a conservative estimate without tokenizing $label', ({ text }) => {+ const count = jest.spyOn(AiTokenizer.prototype, 'count');+ try {+ expect(Tokenizer.getTokenCount(text, 'o200k_base')).toBe(Buffer.byteLength(text, 'utf8'));+ expect(count).not.toHaveBeenCalled();+ } finally {+ count.mockRestore();+ }+ });});describe('createExactTokenCounter', () => {@@ -68,5 +88,17 @@ describe('Tokenizer', () => {await Tokenizer.initEncoding('o200k_base');}});++ it('uses a conservative estimate for unsafe input without invoking the tokenizer', async () => {+ const counter = await Tokenizer.createExactTokenCounter('o200k_base');+ const count = jest.spyOn(AiTokenizer.prototype, 'count');+ try {+ const text = '_'.repeat(4 * 1024 + 1);+ expect(counter(text)).toBe(Buffer.byteLength(text, 'utf8'));+ expect(count).not.toHaveBeenCalled();+ } finally {+ count.mockRestore();+ }+ });});});diff --git a/packages/api/src/utils/tokenizer.ts b/packages/api/src/utils/tokenizer.tsindex 83a3c2691f..9bbd9591ee 100644--- a/packages/api/src/utils/tokenizer.ts+++ b/packages/api/src/utils/tokenizer.ts@@ -5,6 +5,20 @@ export type EncodingName = 'o200k_base' | 'claude';type EncodingData = ConstructorParameters<typeof AiTokenizer>[0];+const MAX_TOKENIZER_INPUT_LENGTH = 4 * 1024;++function estimateBoundedTokenCount(text: string): number {+ return Buffer.byteLength(text, 'utf8');+}++function estimateUnavailableTokenCount(text: string): number {+ return Math.ceil(text.length / 4);+}++function requiresTokenEstimate(text: string): boolean {+ return text.length > MAX_TOKENIZER_INPUT_LENGTH;+}+class Tokenizer {private tokenizersCache: Partial<Record<EncodingName, AiTokenizer>> = {};private loadingPromises: Partial<Record<EncodingName, Promise<void>>> = {};@@ -27,7 +41,7 @@ class Tokenizer {return this.loadingPromises[encoding];}- /** Returns a counter that never substitutes an estimate for tokenizer errors. */+ /** Returns a counter that avoids expensive tokenization for oversized content. */async createExactTokenCounter(encoding: EncodingName): Promise<(text: string) => number> {await this.initEncoding(encoding);const tokenizer = this.tokenizersCache[encoding];@@ -35,6 +49,9 @@ class Tokenizer {throw new Error(`Tokenizer encoding failed to initialize: ${encoding}`);}return (text: string): number => {+ if (requiresTokenEstimate(text)) {+ return estimateBoundedTokenCount(text);+ }try {return tokenizer.count(text);} catch (error) {@@ -45,16 +62,19 @@ class Tokenizer {}getTokenCount(text: string, encoding: EncodingName = 'o200k_base'): number {+ if (requiresTokenEstimate(text)) {+ return estimateBoundedTokenCount(text);+ }const tokenizer = this.tokenizersCache[encoding];if (!tokenizer) {this.initEncoding(encoding);- return Math.ceil(text.length / 4);+ return estimateUnavailableTokenCount(text);}try {return tokenizer.count(text);} catch (error) {this.handleCountError(encoding, tokenizer, error);- return Math.ceil(text.length / 4);+ return estimateUnavailableTokenCount(text);}}