Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🚀 feat: Agent Cache Tokens & Anthropic Reasoning Support (#6098)
Danny Avila980262984837
diff --git a/api/app/clients/AnthropicClient.js b/api/app/clients/AnthropicClient.jsindex a2ab752bc2..19f4a3930a 100644--- a/api/app/clients/AnthropicClient.js+++ b/api/app/clients/AnthropicClient.js@@ -746,15 +746,6 @@ class AnthropicClient extends BaseClient {metadata,};- if (!/claude-3[-.]7/.test(model)) {- if (top_p !== undefined) {- requestOptions.top_p = top_p;- }- if (top_k !== undefined) {- requestOptions.top_k = top_k;- }- }-if (this.useMessages) {requestOptions.messages = payload;requestOptions.max_tokens =@@ -769,6 +760,14 @@ class AnthropicClient extends BaseClient {thinkingBudget: this.options.thinkingBudget,});+ if (!/claude-3[-.]7/.test(model)) {+ requestOptions.top_p = top_p;+ requestOptions.top_k = top_k;+ } else if (requestOptions.thinking == null) {+ requestOptions.topP = top_p;+ requestOptions.topK = top_k;+ }+if (this.systemMessage && this.supportsCacheControl === true) {requestOptions.system = [{diff --git a/api/app/clients/OpenAIClient.js b/api/app/clients/OpenAIClient.jsindex 8d0bce25d2..4bc2d66ca0 100644--- a/api/app/clients/OpenAIClient.js+++ b/api/app/clients/OpenAIClient.js@@ -1309,6 +1309,12 @@ ${convo}modelOptions.include_reasoning = true;reasoningKey = 'reasoning';}+ if (this.useOpenRouter && modelOptions.reasoning_effort != null) {+ modelOptions.reasoning = {+ effort: modelOptions.reasoning_effort,+ };+ delete modelOptions.reasoning_effort;+ }this.streamHandler = new SplitStreamHandler({reasoningKey,diff --git a/api/app/clients/specs/AnthropicClient.test.js b/api/app/clients/specs/AnthropicClient.test.jsindex b565e6d188..223f3038c0 100644--- a/api/app/clients/specs/AnthropicClient.test.js+++ b/api/app/clients/specs/AnthropicClient.test.js@@ -680,4 +680,53 @@ describe('AnthropicClient', () => {expect(capturedOptions).not.toHaveProperty('top_p');});});++ it('should include top_k and top_p parameters for Claude-3.7 models when thinking is explicitly disabled', async () => {+ const client = new AnthropicClient('test-api-key', {+ modelOptions: {+ model: 'claude-3-7-sonnet',+ temperature: 0.7,+ topK: 10,+ topP: 0.9,+ },+ thinking: false,+ });++ async function* mockAsyncGenerator() {+ yield { type: 'message_start', message: { usage: {} } };+ yield { delta: { text: 'Test response' } };+ yield { type: 'message_delta', usage: {} };+ }++ jest.spyOn(client, 'createResponse').mockImplementation(() => {+ return mockAsyncGenerator();+ });++ let capturedOptions = null;+ jest.spyOn(client, 'getClient').mockImplementation((options) => {+ capturedOptions = options;+ return {};+ });++ const payload = [{ role: 'user', content: 'Test message' }];+ await client.sendCompletion(payload, {});++ expect(capturedOptions).toHaveProperty('topK', 10);+ expect(capturedOptions).toHaveProperty('topP', 0.9);++ client.setOptions({+ modelOptions: {+ model: 'claude-3.7-sonnet',+ temperature: 0.7,+ topK: 10,+ topP: 0.9,+ },+ thinking: false,+ });++ await client.sendCompletion(payload, {});++ expect(capturedOptions).toHaveProperty('topK', 10);+ expect(capturedOptions).toHaveProperty('topP', 0.9);+ });});diff --git a/api/package.json b/api/package.jsonindex e386394acb..2d83fddcbd 100644--- a/api/package.json+++ b/api/package.json@@ -45,7 +45,7 @@"@langchain/google-genai": "^0.1.9","@langchain/google-vertexai": "^0.2.0","@langchain/textsplitters": "^0.1.0",- "@librechat/agents": "^2.1.3",+ "@librechat/agents": "^2.1.7","@waylaidwanderer/fetch-event-source": "^3.0.1","axios": "1.7.8","bcryptjs": "^2.4.3",diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.jsindex bff7dc65eb..99d64bb9a6 100644--- a/api/server/controllers/agents/client.js+++ b/api/server/controllers/agents/client.js@@ -27,10 +27,10 @@ const {formatContentStrings,createContextHandlers,} = require('~/app/clients/prompts');-const { encodeAndFormat } = require('~/server/services/Files/images/encode');+const { spendTokens, spendStructuredTokens } = require('~/models/spendTokens');const { getBufferString, HumanMessage } = require('@langchain/core/messages');+const { encodeAndFormat } = require('~/server/services/Files/images/encode');const Tokenizer = require('~/server/services/Tokenizer');-const { spendTokens } = require('~/models/spendTokens');const BaseClient = require('~/app/clients/BaseClient');const { createRun } = require('./run');const { logger } = require('~/config');@@ -380,15 +380,34 @@ class AgentClient extends BaseClient {if (!collectedUsage || !collectedUsage.length) {return;}- const input_tokens = collectedUsage[0]?.input_tokens || 0;+ const input_tokens =+ (collectedUsage[0]?.input_tokens || 0) ++ (Number(collectedUsage[0]?.input_token_details?.cache_creation) || 0) ++ (Number(collectedUsage[0]?.input_token_details?.cache_read) || 0);let output_tokens = 0;let previousTokens = input_tokens; // Start with original inputfor (let i = 0; i < collectedUsage.length; i++) {const usage = collectedUsage[i];+ if (!usage) {+ continue;+ }++ const cache_creation = Number(usage.input_token_details?.cache_creation) || 0;+ const cache_read = Number(usage.input_token_details?.cache_read) || 0;++ const txMetadata = {+ context,+ conversationId: this.conversationId,+ user: this.user ?? this.options.req.user?.id,+ endpointTokenConfig: this.options.endpointTokenConfig,+ model: usage.model ?? model ?? this.model ?? this.options.agent.model_parameters.model,+ };+if (i > 0) {// Count new tokens generated (input_tokens minus previous accumulated tokens)- output_tokens += (Number(usage.input_tokens) || 0) - previousTokens;+ output_tokens +=+ (Number(usage.input_tokens) || 0) + cache_creation + cache_read - previousTokens;}// Add this message's output tokens@@ -396,16 +415,26 @@ class AgentClient extends BaseClient {// Update previousTokens to include this message's outputpreviousTokens += Number(usage.output_tokens) || 0;- spendTokens(- {- context,- conversationId: this.conversationId,- user: this.user ?? this.options.req.user?.id,- endpointTokenConfig: this.options.endpointTokenConfig,- model: usage.model ?? model ?? this.model ?? this.options.agent.model_parameters.model,- },- { promptTokens: usage.input_tokens, completionTokens: usage.output_tokens },- ).catch((err) => {++ if (cache_creation > 0 || cache_read > 0) {+ spendStructuredTokens(txMetadata, {+ promptTokens: {+ input: usage.input_tokens,+ write: cache_creation,+ read: cache_read,+ },+ completionTokens: usage.output_tokens,+ }).catch((err) => {+ logger.error(+ '[api/server/controllers/agents/client.js #recordCollectedUsage] Error spending structured tokens',+ err,+ );+ });+ }+ spendTokens(txMetadata, {+ promptTokens: usage.input_tokens,+ completionTokens: usage.output_tokens,+ }).catch((err) => {logger.error('[api/server/controllers/agents/client.js #recordCollectedUsage] Error spending tokens',err,@@ -792,7 +821,10 @@ class AgentClient extends BaseClient {throw new Error('Run not initialized');}const { handleLLMEnd, collected: collectedMetadata } = createMetadataAggregator();- const clientOptions = {};+ /** @type {import('@librechat/agents').ClientOptions} */+ const clientOptions = {+ maxTokens: 75,+ };const providerConfig = this.options.req.app.locals[this.options.agent.provider];if (providerConfig &&diff --git a/api/server/services/Endpoints/agents/title.js b/api/server/services/Endpoints/agents/title.jsindex 56fd28668d..f25746582e 100644--- a/api/server/services/Endpoints/agents/title.js+++ b/api/server/services/Endpoints/agents/title.js@@ -20,10 +20,19 @@ const addTitle = async (req, { text, response, client }) => {const titleCache = getLogStores(CacheKeys.GEN_TITLE);const key = `${req.user.id}-${response.conversationId}`;+ const responseText =+ response?.content && Array.isArray(response?.content)+ ? response.content.reduce((acc, block) => {+ if (block?.type === 'text') {+ return acc + block.text;+ }+ return acc;+ }, '')+ : (response?.content ?? response?.text ?? '');const title = await client.titleConvo({text,- responseText: response?.text ?? '',+ responseText,conversationId: response.conversationId,});await titleCache.set(key, title, 120000);diff --git a/api/server/services/Endpoints/anthropic/llm.js b/api/server/services/Endpoints/anthropic/llm.jsindex 186444cec8..9f20b8e61d 100644--- a/api/server/services/Endpoints/anthropic/llm.js+++ b/api/server/services/Endpoints/anthropic/llm.js@@ -1,6 +1,6 @@const { HttpsProxyAgent } = require('https-proxy-agent');const { anthropicSettings, removeNullishValues } = require('librechat-data-provider');-const { checkPromptCacheSupport, getClaudeHeaders } = require('./helpers');+const { checkPromptCacheSupport, getClaudeHeaders, configureReasoning } = require('./helpers');/*** Generates configuration options for creating an Anthropic language model (LLM) instance.@@ -49,13 +49,14 @@ function getLLMConfig(apiKey, options = {}) {clientOptions: {},};+ requestOptions = configureReasoning(requestOptions, systemOptions);+if (!/claude-3[-.]7/.test(mergedOptions.model)) {- if (mergedOptions.topP !== undefined) {- requestOptions.topP = mergedOptions.topP;- }- if (mergedOptions.topK !== undefined) {- requestOptions.topK = mergedOptions.topK;- }+ requestOptions.topP = mergedOptions.topP;+ requestOptions.topK = mergedOptions.topK;+ } else if (requestOptions.thinking == null) {+ requestOptions.topP = mergedOptions.topP;+ requestOptions.topK = mergedOptions.topK;}const supportsCacheControl =diff --git a/api/server/services/Endpoints/anthropic/llm.spec.js b/api/server/services/Endpoints/anthropic/llm.spec.jsindex a1dc6a44b6..9c453efb92 100644--- a/api/server/services/Endpoints/anthropic/llm.spec.js+++ b/api/server/services/Endpoints/anthropic/llm.spec.js@@ -109,4 +109,45 @@ describe('getLLMConfig', () => {// Just verifying that the promptCache setting is processedexpect(result.llmConfig).toBeDefined();});++ it('should include topK and topP for Claude-3.7 models when thinking is not enabled', () => {+ // Test with thinking explicitly set to null/undefined+ const result = getLLMConfig('test-api-key', {+ modelOptions: {+ model: 'claude-3-7-sonnet',+ topK: 10,+ topP: 0.9,+ thinking: false,+ },+ });++ expect(result.llmConfig).toHaveProperty('topK', 10);+ expect(result.llmConfig).toHaveProperty('topP', 0.9);++ // Test with thinking explicitly set to false+ const result2 = getLLMConfig('test-api-key', {+ modelOptions: {+ model: 'claude-3-7-sonnet',+ topK: 10,+ topP: 0.9,+ thinking: false,+ },+ });++ expect(result2.llmConfig).toHaveProperty('topK', 10);+ expect(result2.llmConfig).toHaveProperty('topP', 0.9);++ // Test with decimal notation as well+ const result3 = getLLMConfig('test-api-key', {+ modelOptions: {+ model: 'claude-3.7-sonnet',+ topK: 10,+ topP: 0.9,+ thinking: false,+ },+ });++ expect(result3.llmConfig).toHaveProperty('topK', 10);+ expect(result3.llmConfig).toHaveProperty('topP', 0.9);+ });});diff --git a/api/server/services/Endpoints/openAI/llm.js b/api/server/services/Endpoints/openAI/llm.jsindex c12f835f2f..0fa899b4a3 100644--- a/api/server/services/Endpoints/openAI/llm.js+++ b/api/server/services/Endpoints/openAI/llm.js@@ -29,7 +29,6 @@ function getLLMConfig(apiKey, options = {}) {const {modelOptions = {},reverseProxyUrl,- useOpenRouter,defaultQuery,headers,proxy,@@ -56,9 +55,11 @@ function getLLMConfig(apiKey, options = {}) {});}+ let useOpenRouter;/** @type {OpenAIClientOptions['configuration']} */const configOptions = {};- if (useOpenRouter || (reverseProxyUrl && reverseProxyUrl.includes(KnownEndpoints.openrouter))) {+ if (reverseProxyUrl && reverseProxyUrl.includes(KnownEndpoints.openrouter)) {+ useOpenRouter = true;llmConfig.include_reasoning = true;configOptions.baseURL = reverseProxyUrl;configOptions.defaultHeaders = Object.assign(@@ -118,6 +119,13 @@ function getLLMConfig(apiKey, options = {}) {llmConfig.organization = process.env.OPENAI_ORGANIZATION;}+ if (useOpenRouter && llmConfig.reasoning_effort != null) {+ llmConfig.reasoning = {+ effort: llmConfig.reasoning_effort,+ };+ delete llmConfig.reasoning_effort;+ }+return {/** @type {OpenAIClientOptions} */llmConfig,diff --git a/client/src/components/Chat/Messages/Content/ContentParts.tsx b/client/src/components/Chat/Messages/Content/ContentParts.tsxindex b997060c61..ddf08976eb 100644--- a/client/src/components/Chat/Messages/Content/ContentParts.tsx+++ b/client/src/components/Chat/Messages/Content/ContentParts.tsx@@ -109,7 +109,9 @@ const ContentParts = memo(return val;})}- label={isSubmitting ? localize('com_ui_thinking') : localize('com_ui_thoughts')}+ label={+ isSubmitting && isLast ? localize('com_ui_thinking') : localize('com_ui_thoughts')+ }/></div>)}diff --git a/package-lock.json b/package-lock.jsonindex c513481bf4..e4e9d06680 100644--- a/package-lock.json+++ b/package-lock.json@@ -61,7 +61,7 @@"@langchain/google-genai": "^0.1.9","@langchain/google-vertexai": "^0.2.0","@langchain/textsplitters": "^0.1.0",- "@librechat/agents": "^2.1.3",+ "@librechat/agents": "^2.1.7","@waylaidwanderer/fetch-event-source": "^3.0.1","axios": "1.7.8","bcryptjs": "^2.4.3",@@ -15984,9 +15984,9 @@}},"node_modules/@librechat/agents": {- "version": "2.1.3",- "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-2.1.3.tgz",- "integrity": "sha512-4pPkLpjhA3DDiZQOULcrpbdQaOBC4JuUMdcVTUyYBHcA63SJT3olstmRQkGKNvoXLFLeQyJ0jkOqkEpzLJzk/g==",+ "version": "2.1.7",+ "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-2.1.7.tgz",+ "integrity": "sha512-/+AvxH75K0dSSUeHqT8jPZCcqcQUWdB56g9ls7ho0Nw9vdxfezBhF/hXnOk5oORHeEXlGEKNE6YPyjAhCmNIOg==","dependencies": {"@aws-crypto/sha256-js": "^5.2.0","@aws-sdk/credential-provider-node": "^3.613.0",