Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🧮 perf: Enable Agent Context Count Reuse (#15130)
Danny Avila77cb72e50c4d
diff --git a/api/package.json b/api/package.jsonindex 043fcdd7ee..44bf9127c0 100644--- a/api/package.json+++ b/api/package.json@@ -46,7 +46,7 @@"@azure/storage-blob": "^12.30.0","@google/genai": "^2.8.0","@keyv/redis": "5.1.6",- "@librechat/agents": "^3.6.12",+ "@librechat/agents": "^3.6.13","@librechat/api": "*","@librechat/data-schemas": "*","@microsoft/microsoft-graph-client": "^3.0.7",diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.jsindex cb37cd6a31..0332bdddc1 100644--- a/api/server/controllers/agents/client.js+++ b/api/server/controllers/agents/client.js@@ -17,7 +17,7 @@ const {omitTitleOptions,getProviderConfig,memoryInstructions,- createTokenCounter,+ createCachedTokenCounter,applyContextToAgent,isMemoryAgentEnabled,recordCollectedUsage,@@ -3249,7 +3249,7 @@ class AgentClient extends BaseClient {};const toolSet = buildToolSet(this.options.agent);- const tokenCounter = createTokenCounter(this.getEncoding());+ const tokenCounter = await createCachedTokenCounter(this.getEncoding());/** Pre-resolve invoked skill bodies + re-prime files before formatting messages */const skillPrimeResult = this.options.primeInvokedSkills@@ -3889,7 +3889,7 @@ class AgentClient extends BaseClient {this.contentParts.push(...seedContent);}- const tokenCounter = createTokenCounter(this.getEncoding());+ const tokenCounter = await createCachedTokenCounter(this.getEncoding());const agents = collectReachableAgents([this.options.agent,...(this.agentConfigs?.size > 0 ? this.agentConfigs.values() : []),diff --git a/api/server/controllers/agents/client.test.js b/api/server/controllers/agents/client.test.jsindex 37d4b54094..027feffe12 100644--- a/api/server/controllers/agents/client.test.js+++ b/api/server/controllers/agents/client.test.js@@ -47,7 +47,7 @@ jest.mock('@librechat/api', () => ({createRun: (...args) => mockCreateRun(...args),countFormattedMessageTokens: jest.fn(() => 42),countTokens: jest.fn((text) => Math.ceil(String(text ?? '').length / 4)),- createTokenCounter: jest.fn(() => jest.fn(() => 0)),+ createCachedTokenCounter: jest.fn(async () => jest.fn(() => 0)),createDetachedSubagentUsageRecorder: (...args) =>mockCreateDetachedSubagentUsageRecorder(...args),captureAgentCheckpointGeneration: (...args) => mockCaptureAgentCheckpointGeneration(...args),diff --git a/package-lock.json b/package-lock.jsonindex c0c365f79d..28e06d1b14 100644--- a/package-lock.json+++ b/package-lock.json@@ -63,7 +63,7 @@"@azure/storage-blob": "^12.30.0","@google/genai": "^2.8.0","@keyv/redis": "5.1.6",- "@librechat/agents": "^3.6.12",+ "@librechat/agents": "^3.6.13","@librechat/api": "*","@librechat/data-schemas": "*","@microsoft/microsoft-graph-client": "^3.0.7",@@ -10630,9 +10630,9 @@}},"node_modules/@librechat/agents": {- "version": "3.6.12",- "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-3.6.12.tgz",- "integrity": "sha512-S73bRfCJ4bklASObib7gpSn9Vm0IkJ9ZmVgtzcKzUYju8lkb18vU9FcK1ZFcyZl7/FZAQbHIHRYBDKHMqEn7xg==",+ "version": "3.6.13",+ "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-3.6.13.tgz",+ "integrity": "sha512-nrcxEKwJDmIDsGf7txLqTgqqSKkJKl90xS5vqTtAoYSi4/EVPZ6ZFpJHLT3WMn2nWL5jjxdOmstet3JbSF1B+A==","license": "MIT","dependencies": {"@anthropic-ai/sdk": "^0.115.0",@@ -42822,7 +42822,7 @@"@azure/storage-blob": "^12.30.0","@google/genai": "^2.8.0","@keyv/redis": "5.1.6",- "@librechat/agents": "^3.6.12",+ "@librechat/agents": "^3.6.13","@librechat/data-schemas": "*","@modelcontextprotocol/sdk": "^1.30.0","@opentelemetry/api": "^1.9.0",diff --git a/packages/api/package.json b/packages/api/package.jsonindex b6006d72af..1b18249938 100644--- a/packages/api/package.json+++ b/packages/api/package.json@@ -113,7 +113,7 @@"@azure/storage-blob": "^12.30.0","@google/genai": "^2.8.0","@keyv/redis": "5.1.6",- "@librechat/agents": "^3.6.12",+ "@librechat/agents": "^3.6.13","@librechat/data-schemas": "*","@modelcontextprotocol/sdk": "^1.30.0","@opentelemetry/api": "^1.9.0",diff --git a/packages/api/src/agents/client.spec.ts b/packages/api/src/agents/client.spec.tsindex c62cb09662..fa62cb4bb1 100644--- a/packages/api/src/agents/client.spec.ts+++ b/packages/api/src/agents/client.spec.ts@@ -1,6 +1,10 @@import { ContentTypes } from 'librechat-data-provider';+import { Tokenizer as AiTokenizer } from 'ai-tokenizer';+import { Providers, StandardGraph } from '@librechat/agents';+import { HumanMessage } from '@librechat/agents/langchain/messages';import type { TMessage } from 'librechat-data-provider';import {+ createCachedTokenCounter,prependQuotes,prependFileContext,applyAttachmentOnlyText,@@ -8,6 +12,39 @@ import {} from './client';import { ATTACHMENT_ONLY_TEXT } from '~/files/context';+describe('createCachedTokenCounter', () => {+ it('enables stable-message reuse in the agents runtime', async () => {+ const getTokenCount = jest.spyOn(AiTokenizer.prototype, 'count');+ try {+ const tokenCounter = await createCachedTokenCounter('o200k_base');+ const graph = new StandardGraph({+ runId: 'token-cache-integration',+ agents: [+ {+ agentId: 'primary',+ provider: Providers.OPENAI,+ instructions: 'Test instructions',+ },+ ],+ tokenCounter,+ });+ const agentContext = graph.agentContexts.get('primary');+ await agentContext?.tokenCalculationPromise;+ getTokenCount.mockClear();+ const message = new HumanMessage('Stable retained context');++ agentContext?.contextPressureTokenCounts?.count(message);+ const callsAfterFirstCount = getTokenCount.mock.calls.length;+ agentContext?.contextPressureTokenCounts?.count(message);++ expect(callsAfterFirstCount).toBeGreaterThan(0);+ expect(getTokenCount).toHaveBeenCalledTimes(callsAfterFirstCount);+ } finally {+ getTokenCount.mockRestore();+ }+ });+});+describe('prependFileContext', () => {it('prepends file context to string content', () => {const message: FormattedMessageWithContent = { content: 'Answer this question.' };diff --git a/packages/api/src/agents/client.ts b/packages/api/src/agents/client.tsindex bc7ca9c404..fe00e5ec2a 100644--- a/packages/api/src/agents/client.ts+++ b/packages/api/src/agents/client.ts@@ -6,9 +6,10 @@ import {getTokenCountForMessage,estimateOpenAIImageTokens,estimateAnthropicImageTokens,+ markTokenCounterCacheCompatible,} from '@librechat/agents';+import type { MessageContentComplex, TokenCounter } from '@librechat/agents';import type { BaseMessage } from '@librechat/agents/langchain/messages';-import type { MessageContentComplex } from '@librechat/agents';import type { Agent, TMessage } from 'librechat-data-provider';import type { ServerRequest } from '~/types';import { getSafeErrorMetadata, mergeQuotedText, formatQuotesAsMarkdown } from '~/utils';@@ -378,9 +379,19 @@ export function countFormattedMessageTokens(return isClaude ? Math.ceil(numTokens * CLAUDE_TOKEN_CORRECTION) : numTokens;}-export function createTokenCounter(encoding: Parameters<typeof Tokenizer.getTokenCount>[1]) {+export function createTokenCounter(+ encoding: Parameters<typeof Tokenizer.getTokenCount>[1],+): TokenCounter {+ return createMessageTokenCounter(encoding, (text: string) =>+ Tokenizer.getTokenCount(text, encoding),+ );+}++function createMessageTokenCounter(+ encoding: Parameters<typeof Tokenizer.getTokenCount>[1],+ countTokens: (text: string) => number,+): TokenCounter {const isClaude = encoding === 'claude';- const countTokens = (text: string) => Tokenizer.getTokenCount(text, encoding);return function (message: BaseMessage): number {const count = getTokenCountForMessage(message,@@ -391,6 +402,13 @@ export function createTokenCounter(encoding: Parameters<typeof Tokenizer.getToke};}+export async function createCachedTokenCounter(+ encoding: Parameters<typeof Tokenizer.getTokenCount>[1],+): Promise<TokenCounter> {+ const countTokens = await Tokenizer.createExactTokenCounter(encoding ?? 'o200k_base');+ return markTokenCounterCacheCompatible(createMessageTokenCounter(encoding, countTokens));+}+export function logToolError(_graph: unknown, error: unknown, toolId: string): void {/*** A GraphInterrupt unwinding out of a tool body is the HITL pause working asdiff --git a/packages/api/src/utils/tokenizer.spec.ts b/packages/api/src/utils/tokenizer.spec.tsindex b8c1bd8d98..d2a4ccb6a5 100644--- a/packages/api/src/utils/tokenizer.spec.ts+++ b/packages/api/src/utils/tokenizer.spec.ts@@ -1,3 +1,4 @@+import { Tokenizer as AiTokenizer } from 'ai-tokenizer';import Tokenizer from './tokenizer';jest.mock('@librechat/data-schemas', () => ({@@ -53,4 +54,19 @@ describe('Tokenizer', () => {expect(count).toBeGreaterThan(0);});});++ describe('createExactTokenCounter', () => {+ it('throws instead of returning a fallback estimate after a tokenizer failure', async () => {+ const counter = await Tokenizer.createExactTokenCounter('o200k_base');+ const count = jest.spyOn(AiTokenizer.prototype, 'count').mockImplementationOnce(() => {+ throw new Error('tokenizer failed');+ });+ try {+ expect(() => counter('Stable cached context')).toThrow('tokenizer failed');+ } finally {+ count.mockRestore();+ await Tokenizer.initEncoding('o200k_base');+ }+ });+ });});diff --git a/packages/api/src/utils/tokenizer.ts b/packages/api/src/utils/tokenizer.tsindex d66908712a..83a3c2691f 100644--- a/packages/api/src/utils/tokenizer.ts+++ b/packages/api/src/utils/tokenizer.ts@@ -27,6 +27,23 @@ class Tokenizer {return this.loadingPromises[encoding];}+ /** Returns a counter that never substitutes an estimate for tokenizer errors. */+ async createExactTokenCounter(encoding: EncodingName): Promise<(text: string) => number> {+ await this.initEncoding(encoding);+ const tokenizer = this.tokenizersCache[encoding];+ if (!tokenizer) {+ throw new Error(`Tokenizer encoding failed to initialize: ${encoding}`);+ }+ return (text: string): number => {+ try {+ return tokenizer.count(text);+ } catch (error) {+ this.handleCountError(encoding, tokenizer, error);+ throw error;+ }+ };+ }+getTokenCount(text: string, encoding: EncodingName = 'o200k_base'): number {const tokenizer = this.tokenizersCache[encoding];if (!tokenizer) {@@ -36,13 +53,22 @@ class Tokenizer {try {return tokenizer.count(text);} catch (error) {- logger.error('[Tokenizer] Error getting token count:', error);- delete this.tokenizersCache[encoding];- delete this.loadingPromises[encoding];- this.initEncoding(encoding);+ this.handleCountError(encoding, tokenizer, error);return Math.ceil(text.length / 4);}}++ private handleCountError(encoding: EncodingName, tokenizer: AiTokenizer, error: unknown): void {+ logger.error('[Tokenizer] Error getting token count:', error);+ if (this.tokenizersCache[encoding] !== tokenizer) {+ return;+ }+ delete this.tokenizersCache[encoding];+ delete this.loadingPromises[encoding];+ void this.initEncoding(encoding).catch((initError: unknown) => {+ logger.error(`[Tokenizer] Error reloading ${encoding} encoding:`, initError);+ });+ }}const TokenizerSingleton: Tokenizer = new Tokenizer();