Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🪚 fix: Count Large Prompts in Bounded Tokenizer Chunks (#16305)

lia-by-librechat[bot]56301c1b5a32
diff --git a/packages/api/src/agents/client.spec.ts b/packages/api/src/agents/client.spec.ts
index 97700e4492..f3a5b56e9d 100644
--- a/packages/api/src/agents/client.spec.ts
+++ b/packages/api/src/agents/client.spec.ts
@@ -1,8 +1,11 @@
import { Tokenizer as AiTokenizer } from 'ai-tokenizer';
import { Providers, StandardGraph } from '@librechat/agents';
-import { HumanMessage } from '@librechat/agents/langchain/messages';
+import { HumanMessage, SystemMessage } from '@librechat/agents/langchain/messages';
import { ContentTypes, DEFAULT_MAX_RETAINED_TOOL_COUNT_CHARS } from 'librechat-data-provider';
import type { TMessage } from 'librechat-data-provider';
+import type { LCTool } from '@librechat/agents';
+import type { FormattedMessageWithContent } from './client';
+import type { EncodingName } from '~/utils/tokenizer';
import {
collectToolCallIds,
countRetainedToolTokens,
@@ -10,11 +13,130 @@ import {
prependQuotes,
prependFileContext,
applyAttachmentOnlyText,
- type FormattedMessageWithContent,
} from './client';
import { ATTACHMENT_ONLY_TEXT } from '~/files/context';
+import Tokenizer from '~/utils/tokenizer';
describe('createCachedTokenCounter', () => {
+ const encodings: EncodingName[] = ['o200k_base', 'claude'];
+
+ it.each(encodings)('counts SDK-sized slices accurately with %s', async (encoding) => {
+ const counter = await createCachedTokenCounter(encoding);
+ const text = 'word '.repeat(3277).slice(0, 16384);
+ const exact = Tokenizer.countExactTokens(text, encoding)!;
+ const expected = encoding === 'claude' ? Math.ceil(exact * 1.1) : exact;
+ expect(counter(new SystemMessage(text))).toBeGreaterThanOrEqual(expected * 0.99);
+ expect(counter(new SystemMessage(text))).toBeLessThan(expected * 1.02);
+ });
+
+ it.each([false, true])(
+ 'sends fitting instructions with summarization=%s',
+ async (summarizationEnabled) => {
+ const graph = new StandardGraph({
+ runId: `bounded-instructions-${summarizationEnabled}`,
+ agents: [
+ {
+ agentId: 'primary',
+ provider: Providers.OPENAI,
+ instructions: 'word '.repeat(1024),
+ maxContextTokens: 4000,
+ summarizationEnabled,
+ },
+ ],
+ tokenCounter: await createCachedTokenCounter('o200k_base'),
+ });
+ graph.overrideTestModel(['ok']);
+ const result = await graph
+ .createAgentNode('primary')
+ .invoke(
+ { messages: [new HumanMessage('Hi')] },
+ { configurable: { thread_id: graph.runId }, recursionLimit: 12 },
+ );
+ expect(result.messages[result.messages.length - 1]?.content).toBe('ok');
+ expect(graph.agentContexts.get('primary')?.instructionTokens).toBeLessThan(1100);
+ },
+ );
+
+ it('sends a fitting 400-tool programmatic prompt with no custom instructions', async () => {
+ const tools: LCTool[] = Array.from({ length: 400 }, (_, index) => ({
+ name: `lookup_${index}`,
+ description:
+ 'Search the project documents and return matching results with source references. '.repeat(
+ 16,
+ ),
+ parameters: {
+ type: 'object',
+ properties: {
+ query: {
+ type: 'string',
+ description: 'The search query to match against project documents.',
+ },
+ limit: {
+ type: 'number',
+ description: 'The maximum number of matching documents to return.',
+ },
+ },
+ required: ['query'],
+ },
+ allowed_callers: ['code_execution'],
+ }));
+ const graph = new StandardGraph({
+ runId: 'bounded-programmatic-instructions',
+ agents: [
+ {
+ agentId: 'primary',
+ provider: Providers.ANTHROPIC,
+ instructions: '',
+ maxContextTokens: 500000,
+ toolDefinitions: [
+ {
+ name: 'run_tools_with_code',
+ description: 'Execute code',
+ parameters: { type: 'object', properties: { code: { type: 'string' } } },
+ },
+ ...tools,
+ ],
+ },
+ ],
+ tokenCounter: await createCachedTokenCounter('claude'),
+ });
+ graph.overrideTestModel(['ok']);
+ const result = await graph
+ .createAgentNode('primary')
+ .invoke(
+ { messages: [new HumanMessage('Hi')] },
+ { configurable: { thread_id: graph.runId }, recursionLimit: 12 },
+ );
+ expect(result.messages[result.messages.length - 1]?.content).toBe('ok');
+ const context = graph.agentContexts.get('primary');
+ expect(context?.systemMessageTokens).toBeGreaterThan(100000);
+ expect(context?.instructionTokens).toBeLessThan(150000);
+ });
+
+ it('still rejects instructions that genuinely exceed the context budget', async () => {
+ const graph = new StandardGraph({
+ runId: 'oversized-instructions',
+ agents: [
+ {
+ agentId: 'primary',
+ provider: Providers.OPENAI,
+ instructions: 'word '.repeat(5000),
+ maxContextTokens: 4000,
+ },
+ ],
+ tokenCounter: await createCachedTokenCounter('o200k_base'),
+ });
+ graph.overrideTestModel(['should not be called']);
+ await expect(
+ graph
+ .createAgentNode('primary')
+ .invoke(
+ { messages: [new HumanMessage('Hi')] },
+ { configurable: { thread_id: graph.runId }, recursionLimit: 12 },
+ ),
+ ).rejects.toThrow('empty_messages');
+ });
+
it('enables stable-message reuse in the agents runtime', async () => {
const getTokenCount = jest.spyOn(AiTokenizer.prototype, 'count');
try {
@@ -33,7 +155,7 @@ describe('createCachedTokenCounter', () => {
const agentContext = graph.agentContexts.get('primary');
await agentContext?.tokenCalculationPromise;
getTokenCount.mockClear();
- const message = new HumanMessage('Stable retained context');
+ const message = new HumanMessage('Stable retained context '.repeat(400));
agentContext?.contextPressureTokenCounts?.count(message);
const callsAfterFirstCount = getTokenCount.mock.calls.length;
diff --git a/packages/api/src/utils/tokenizer.spec.ts b/packages/api/src/utils/tokenizer.spec.ts
index bda83078ad..feaddeb64a 100644
--- a/packages/api/src/utils/tokenizer.spec.ts
+++ b/packages/api/src/utils/tokenizer.spec.ts
@@ -1,5 +1,6 @@
import { Tokenizer as AiTokenizer } from 'ai-tokenizer';
-import Tokenizer from './tokenizer';
+import type { EncodingName } from './tokenizer';
+import Tokenizer, { countTokens } from './tokenizer';
jest.mock('@librechat/data-schemas', () => ({
logger: {
@@ -32,6 +33,16 @@ describe('Tokenizer', () => {
expect(count).toBeGreaterThan(0);
});
+ it('keeps the oversized cold-start fallback and loads the encoding for the next count', async () => {
+ await jest.isolateModulesAsync(async () => {
+ const cold = (await import('./tokenizer')).default;
+ const text = 'word '.repeat(4096);
+ expect(cold.getTokenCount(text, 'o200k_base')).toBe(Buffer.byteLength(text, 'utf8'));
+ await cold.initEncoding('o200k_base');
+ expect(cold.getTokenCount(text, 'o200k_base')).toBeLessThan(text.length / 4);
+ });
+ });
+
it('should deduplicate concurrent init calls', async () => {
const [, , count] = await Promise.all([
Tokenizer.initEncoding('o200k_base'),
@@ -60,18 +71,29 @@ describe('Tokenizer', () => {
expect(count).toBeGreaterThan(0);
});
- it.each([
- { label: 'an uninterrupted non-whitespace run', text: '_'.repeat(4 * 1024 + 1) },
- { label: 'an oversized input', text: 'word '.repeat(1024) },
- { label: 'multibyte input', text: '界'.repeat(4 * 1024 + 1) },
- ])('uses a conservative estimate without tokenizing $label', ({ text }) => {
- const count = jest.spyOn(AiTokenizer.prototype, 'count');
+ it('uses the same bounded count through the asynchronous public helper', async () => {
+ const text = 'word '.repeat(4096);
+ expect(await countTokens(text)).toBe(Tokenizer.getTokenCount(text, 'o200k_base'));
+ expect(await countTokens(text)).toBeLessThan(text.length / 4);
+ });
+
+ it('uses the established fallback for the whole input after a later chunk fails', async () => {
+ const text = 'word '.repeat(4096);
+ const original = AiTokenizer.prototype.count;
+ const count = jest
+ .spyOn(AiTokenizer.prototype, 'count')
+ .mockImplementationOnce(original)
+ .mockImplementationOnce(() => {
+ throw new Error('second chunk failed');
+ });
try {
expect(Tokenizer.getTokenCount(text, 'o200k_base')).toBe(Buffer.byteLength(text, 'utf8'));
- expect(count).not.toHaveBeenCalled();
+ expect(count).toHaveBeenCalledTimes(2);
} finally {
count.mockRestore();
+ await Tokenizer.initEncoding('o200k_base');
}
+ expect(Tokenizer.getTokenCount(text, 'o200k_base')).toBeLessThan(text.length / 4);
});
});
@@ -89,17 +111,103 @@ describe('Tokenizer', () => {
}
});
- it('uses a conservative estimate for unsafe input without invoking the tokenizer', async () => {
+ it('never caches a partial count after a later chunk fails and recovers on retry', async () => {
const counter = await Tokenizer.createExactTokenCounter('o200k_base');
+ const text = 'word '.repeat(4096);
+ const original = AiTokenizer.prototype.count;
+ const count = jest
+ .spyOn(AiTokenizer.prototype, 'count')
+ .mockImplementationOnce(original)
+ .mockImplementationOnce(() => {
+ throw new Error('second chunk failed');
+ });
+ try {
+ expect(() => counter(text)).toThrow('second chunk failed');
+ expect(count).toHaveBeenCalledTimes(2);
+ } finally {
+ count.mockRestore();
+ await Tokenizer.initEncoding('o200k_base');
+ }
+ const reloaded = await Tokenizer.createExactTokenCounter('o200k_base');
+ expect(counter(text)).toBe(reloaded(text));
+ });
+ });
+
+ const encodings: EncodingName[] = ['o200k_base', 'claude'];
+ describe.each(encodings)('bounded counting with %s', (encoding) => {
+ let counter: (text: string) => number;
+ beforeAll(async () => {
+ counter = await Tokenizer.createExactTokenCounter(encoding);
+ });
+
+ it.each([4095, 4096, 4097, 8192, 8193, 65536])(
+ 'counts %i characters without a byte-count cliff',
+ (length) => {
+ const text = 'word '.repeat(Math.ceil(length / 5)).slice(0, length);
+ const exact = Tokenizer.countExactTokens(text, encoding)!;
+ expect(counter(text)).toBeGreaterThanOrEqual(exact * 0.99);
+ expect(counter(text)).toBeLessThanOrEqual(exact * 1.01);
+ expect(Tokenizer.getTokenCount(text, encoding)).toBe(counter(text));
+ },
+ );
+
+ it.each([
+ {
+ label: 'minified JSON',
+ text: JSON.stringify(
+ Array.from({ length: 200 }, (_, id) => ({
+ id,
+ description: 'Search project documents',
+ parameters: { type: 'object', required: ['query'] },
+ })),
+ ),
+ },
+ { label: 'CJK', text: '界'.repeat(4097) },
+ { label: 'emoji', text: '😀'.repeat(4097) },
+ { label: 'mixed density', text: 'word '.repeat(1024) + '界 '.repeat(2048) },
+ ])('measures all of $label rather than extrapolating a prefix', ({ text }) => {
+ const exact = Tokenizer.countExactTokens(text, encoding)!;
+ expect(counter(text)).toBeGreaterThanOrEqual(exact * 0.98);
+ expect(counter(text)).toBeLessThanOrEqual(exact * 1.02);
+ expect(Tokenizer.getTokenCount(text, encoding)).toBe(counter(text));
+ });
+
+ it.each([
+ { label: 'English', text: 'word '.repeat(4096) },
+ { label: 'uninterrupted punctuation', text: '_'.repeat(8193) },
+ { label: 'uninterrupted letters', text: 'a'.repeat(8193) },
+ { label: 'whitespace', text: ' '.repeat(8193) },
+ { label: 'Unicode whitespace', text: '\u2003'.repeat(8193) },
+ { label: 'CJK', text: '界'.repeat(8193) },
+ {
+ label: 'surrogate at input boundary',
+ text: 'word '.repeat(819) + '😀' + 'word '.repeat(819),
+ },
+ { label: 'surrogate at run boundary', text: 'a'.repeat(255) + '😀' + 'b'.repeat(500) },
+ ])('bounds work without dropping or corrupting $label', ({ text }) => {
const count = jest.spyOn(AiTokenizer.prototype, 'count');
try {
- const text = '_'.repeat(4 * 1024 + 1);
- expect(counter(text)).toBe(Buffer.byteLength(text, 'utf8'));
- expect(count).not.toHaveBeenCalled();
+ counter(text);
+ const chunks = count.mock.calls.map(([chunk]) => chunk);
+ expect(chunks.join('')).toBe(text);
+ expect(chunks.length).toBeGreaterThan(1);
+ for (const chunk of chunks) {
+ expect(chunk.length).toBeLessThanOrEqual(4096);
+ expect(chunk.length).toBeGreaterThan(0);
+ expect(/^[\uDC00-\uDFFF]|[\uD800-\uDBFF]$/u.test(chunk)).toBe(false);
+ for (const run of chunk.match(/\s+|\S+/gu) ?? []) {
+ expect(run.length).toBeLessThanOrEqual(256);
+ }
+ }
} finally {
count.mockRestore();
}
});
+
+ it('keeps empty input empty', () => {
+ expect(counter('')).toBe(0);
+ expect(Tokenizer.getTokenCount('', encoding)).toBe(0);
+ });
});
describe('countExactTokens', () => {
@@ -107,11 +215,8 @@ describe('Tokenizer', () => {
await Tokenizer.initEncoding('o200k_base');
});
- it('tokenizes oversized input whole instead of falling back to byte length', () => {
- /** The fast-path estimate is byte length, several times the real count on
- * ordinary text; a figure ADDED to provider accounting cannot carry that.
- * Nor can it carry a sum of slices: a BPE merge spanning a seam is charged
- * twice, so the whole input is tokenized in one pass. */
+ it("tokenizes oversized input whole without the budgeting counter's seam approximation", () => {
+ /** Provider accounting must not carry a sum of slices: BPE merges can change at seams. */
const text = 'word '.repeat(4096);
const exact = Tokenizer.countExactTokens(text, 'o200k_base');
expect(exact).toBeGreaterThan(0);
diff --git a/packages/api/src/utils/tokenizer.ts b/packages/api/src/utils/tokenizer.ts
index 06de186c6d..83148ba86d 100644
--- a/packages/api/src/utils/tokenizer.ts
+++ b/packages/api/src/utils/tokenizer.ts
@@ -6,17 +6,41 @@ export type EncodingName = 'o200k_base' | 'claude';
type EncodingData = ConstructorParameters<typeof AiTokenizer>[0];
const MAX_TOKENIZER_INPUT_LENGTH = 4 * 1024;
+/** Bound BPE's quadratic merge work on uninterrupted text, including whitespace. */
+const MAX_TOKENIZER_RUN_LENGTH = 256;
+const WHITESPACE = /\s/u;
-function estimateBoundedTokenCount(text: string): number {
- return Buffer.byteLength(text, 'utf8');
+/** Near-exact budgeting count; seams can change BPE merges, but no content is omitted. */
+function countBoundedTokens(text: string, tokenizer: AiTokenizer): number {
+ let tokens = 0;
+ for (let offset = 0; offset < text.length; ) {
+ let end = Math.min(offset + MAX_TOKENIZER_INPUT_LENGTH, text.length);
+ let runLength = 0;
+ let previousWhitespace = false;
+ for (let index = offset; index < end; index++) {
+ const whitespace = WHITESPACE.test(text[index]);
+ runLength = whitespace === previousWhitespace ? runLength + 1 : 1;
+ previousWhitespace = whitespace;
+ if (runLength >= MAX_TOKENIZER_RUN_LENGTH) {
+ end = index + 1;
+ break;
+ }
+ }
+ const last = text.charCodeAt(end - 1);
+ const next = text.charCodeAt(end);
+ if (last >= 0xd800 && last <= 0xdbff && next >= 0xdc00 && next <= 0xdfff) {
+ end--;
+ }
+ tokens += tokenizer.count(text.slice(offset, end));
+ offset = end;
+ }
+ return tokens;
}
function estimateUnavailableTokenCount(text: string): number {
- return Math.ceil(text.length / 4);
-}
-
-function requiresTokenEstimate(text: string): boolean {
- return text.length > MAX_TOKENIZER_INPUT_LENGTH;
+ return text.length > MAX_TOKENIZER_INPUT_LENGTH
+ ? Buffer.byteLength(text, 'utf8')
+ : Math.ceil(text.length / 4);
}
class Tokenizer {
@@ -41,7 +65,7 @@ class Tokenizer {
return this.loadingPromises[encoding];
}
- /** Returns a counter that avoids expensive tokenization for oversized content. */
+ /** Returns an initialized, deterministic budgeting counter that throws on tokenizer failure. */
async createExactTokenCounter(encoding: EncodingName): Promise<(text: string) => number> {
await this.initEncoding(encoding);
const tokenizer = this.tokenizersCache[encoding];
@@ -49,11 +73,8 @@ class Tokenizer {
throw new Error(`Tokenizer encoding failed to initialize: ${encoding}`);
}
return (text: string): number => {
- if (requiresTokenEstimate(text)) {
- return estimateBoundedTokenCount(text);
- }
try {
- return tokenizer.count(text);
+ return countBoundedTokens(text, tokenizer);
} catch (error) {
this.handleCountError(encoding, tokenizer, error);
throw error;
@@ -62,16 +83,13 @@ class Tokenizer {
}
getTokenCount(text: string, encoding: EncodingName = 'o200k_base'): number {
- if (requiresTokenEstimate(text)) {
- return estimateBoundedTokenCount(text);
- }
const tokenizer = this.tokenizersCache[encoding];
if (!tokenizer) {
this.initEncoding(encoding);
return estimateUnavailableTokenCount(text);
}
try {
- return tokenizer.count(text);
+ return countBoundedTokens(text, tokenizer);
} catch (error) {
this.handleCountError(encoding, tokenizer, error);
return estimateUnavailableTokenCount(text);
@@ -83,13 +101,11 @@ class Tokenizer {
* cannot be produced, so the caller can omit a figure instead of publishing a
* guess. Used for a value that is ADDED to exact provider accounting — see the
* retained tool results of a turn stopped at the tool-call limit — where
- * {@link getTokenCount}'s two fallbacks would corrupt the sum: the cold-start
- * estimate is a character ratio, and the oversized-input shortcut returns byte
- * length, several times the real count on ordinary text.
+ * {@link getTokenCount}'s cold-start/error estimate and chunk-boundary
+ * approximation must not be added to exact provider accounting.
*
* The input is tokenized whole. Counting it in slices would be cheaper but not
- * exact — a BPE merge spanning a seam is charged twice, ~1 token per 4 KiB
- * measured — and this is the one caller that needs the real number. It costs
+ * exact — seams can change BPE merges — and this caller needs the real number. It costs
* ~60 ms/MB, so the caller decides how much content is worth counting
* (`endpoints.agents.maxRetainedToolCountChars`) rather than a bound here.
*/