Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
💾 feat: Anthropic Prompt Caching (#3670)
Danny Avilaa45b384bbc52
diff --git a/api/app/clients/AnthropicClient.js b/api/app/clients/AnthropicClient.jsindex 3bc33af398..662db65e4a 100644--- a/api/app/clients/AnthropicClient.js+++ b/api/app/clients/AnthropicClient.js@@ -12,12 +12,13 @@ const { encodeAndFormat } = require('~/server/services/Files/images/encode');const {truncateText,formatMessage,+ addCacheControl,titleFunctionPrompt,parseParamFromPrompt,createContextHandlers,} = require('./prompts');-const spendTokens = require('~/models/spendTokens');-const { getModelMaxTokens } = require('~/utils');+const { spendTokens, spendStructuredTokens } = require('~/models/spendTokens');+const { getModelMaxTokens, matchModelName } = require('~/utils');const { sleep } = require('~/server/utils');const BaseClient = require('./BaseClient');const { logger } = require('~/config');@@ -32,6 +33,7 @@ function delayBeforeRetry(attempts, baseDelay = 1000) {return new Promise((resolve) => setTimeout(resolve, baseDelay * attempts));}+const tokenEventTypes = new Set(['message_start', 'message_delta']);const { legacy } = anthropicSettings;class AnthropicClient extends BaseClient {@@ -44,6 +46,24 @@ class AnthropicClient extends BaseClient {? options.contextStrategy.toLowerCase(): 'discard';this.setOptions(options);+ /** @type {string | undefined} */+ this.systemMessage;+ /** @type {AnthropicMessageStartEvent| undefined} */+ this.message_start;+ /** @type {AnthropicMessageDeltaEvent| undefined} */+ this.message_delta;+ /** Whether the model is part of the Claude 3 Family+ * @type {boolean} */+ this.isClaude3;+ /** Whether to use Messages API or Completions API+ * @type {boolean} */+ this.useMessages;+ /** Whether or not the model is limited to the legacy amount of output tokens+ * @type {boolean} */+ this.isLegacyOutput;+ /** Whether or not the model supports Prompt Caching+ * @type {boolean} */+ this.supportsCacheControl;}setOptions(options) {@@ -69,8 +89,10 @@ class AnthropicClient extends BaseClient {model: modelOptions.model || anthropicSettings.model.default,};- this.isClaude3 = this.modelOptions.model.includes('claude-3');- this.isLegacyOutput = !this.modelOptions.model.includes('claude-3-5-sonnet');+ const modelMatch = matchModelName(this.modelOptions.model, EModelEndpoint.anthropic);+ this.isClaude3 = modelMatch.startsWith('claude-3');+ this.isLegacyOutput = !modelMatch.startsWith('claude-3-5-sonnet');+ this.supportsCacheControl = this.checkPromptCacheSupport(modelMatch);if (this.isLegacyOutput &&@@ -147,19 +169,74 @@ class AnthropicClient extends BaseClient {options.baseURL = this.options.reverseProxyUrl;}- if (requestOptions?.model && requestOptions.model.includes('claude-3-5-sonnet')) {+ if (+ this.supportsCacheControl &&+ requestOptions?.model &&+ requestOptions.model.includes('claude-3-5-sonnet')+ ) {options.defaultHeaders = {- 'anthropic-beta': 'max-tokens-3-5-sonnet-2024-07-15',+ 'anthropic-beta': 'max-tokens-3-5-sonnet-2024-07-15,prompt-caching-2024-07-31',+ };+ } else if (this.supportsCacheControl) {+ options.defaultHeaders = {+ 'anthropic-beta': 'prompt-caching-2024-07-31',};}return new Anthropic(options);}- getTokenCountForResponse(response) {+ /**+ * Get stream usage as returned by this client's API response.+ * @returns {AnthropicStreamUsage} The stream usage object.+ */+ getStreamUsage() {+ const inputUsage = this.message_start?.message?.usage ?? {};+ const outputUsage = this.message_delta?.usage ?? {};+ return Object.assign({}, inputUsage, outputUsage);+ }++ /**+ * Calculates the correct token count for the current message based on the token count map and API usage.+ * Edge case: If the calculation results in a negative value, it returns the original estimate.+ * If revisiting a conversation with a chat history entirely composed of token estimates,+ * the cumulative token count going forward should become more accurate as the conversation progresses.+ * @param {Object} params - The parameters for the calculation.+ * @param {Record<string, number>} params.tokenCountMap - A map of message IDs to their token counts.+ * @param {string} params.currentMessageId - The ID of the current message to calculate.+ * @param {AnthropicStreamUsage} params.usage - The usage object returned by the API.+ * @returns {number} The correct token count for the current message.+ */+ calculateCurrentTokenCount({ tokenCountMap, currentMessageId, usage }) {+ const originalEstimate = tokenCountMap[currentMessageId] || 0;++ if (!usage || typeof usage.input_tokens !== 'number') {+ return originalEstimate;+ }++ tokenCountMap[currentMessageId] = 0;+ const totalTokensFromMap = Object.values(tokenCountMap).reduce((sum, count) => {+ const numCount = Number(count);+ return sum + (isNaN(numCount) ? 0 : numCount);+ }, 0);+ const totalInputTokens =+ (usage.input_tokens ?? 0) ++ (usage.cache_creation_input_tokens ?? 0) ++ (usage.cache_read_input_tokens ?? 0);++ const currentMessageTokens = totalInputTokens - totalTokensFromMap;+ return currentMessageTokens > 0 ? currentMessageTokens : originalEstimate;+ }++ /**+ * Get Token Count for LibreChat Message+ * @param {TMessage} responseMessage+ * @returns {number}+ */+ getTokenCountForResponse(responseMessage) {return this.getTokenCountForMessage({role: 'assistant',- content: response.text,+ content: responseMessage.text,});}@@ -212,7 +289,38 @@ class AnthropicClient extends BaseClient {return files;}- async recordTokenUsage({ promptTokens, completionTokens, model, context = 'message' }) {+ /**+ * @param {object} params+ * @param {number} params.promptTokens+ * @param {number} params.completionTokens+ * @param {AnthropicStreamUsage} [params.usage]+ * @param {string} [params.model]+ * @param {string} [params.context='message']+ * @returns {Promise<void>}+ */+ async recordTokenUsage({ promptTokens, completionTokens, usage, model, context = 'message' }) {+ if (usage != null && usage?.input_tokens != null) {+ const input = usage.input_tokens ?? 0;+ const write = usage.cache_creation_input_tokens ?? 0;+ const read = usage.cache_read_input_tokens ?? 0;++ await spendStructuredTokens(+ {+ context,+ user: this.user,+ conversationId: this.conversationId,+ model: model ?? this.modelOptions.model,+ endpointTokenConfig: this.options.endpointTokenConfig,+ },+ {+ promptTokens: { input, write, read },+ completionTokens,+ },+ );++ return;+ }+await spendTokens({context,@@ -560,6 +668,18 @@ class AnthropicClient extends BaseClient {: await client.completions.create(options);}+ /**+ * @param {string} modelName+ * @returns {boolean}+ */+ checkPromptCacheSupport(modelName) {+ const modelMatch = matchModelName(modelName, EModelEndpoint.anthropic);+ if (modelMatch === 'claude-3-5-sonnet' || modelMatch === 'claude-3-haiku') {+ return true;+ }+ return false;+ }+async sendCompletion(payload, { onProgress, abortController }) {if (!abortController) {abortController = new AbortController();@@ -606,10 +726,22 @@ class AnthropicClient extends BaseClient {requestOptions.max_tokens_to_sample = maxOutputTokens || 1500;}- if (this.systemMessage) {+ if (this.systemMessage && this.supportsCacheControl === true) {+ requestOptions.system = [+ {+ type: 'text',+ text: this.systemMessage,+ cache_control: { type: 'ephemeral' },+ },+ ];+ } else if (this.systemMessage) {requestOptions.system = this.systemMessage;}+ if (this.supportsCacheControl === true && this.useMessages) {+ requestOptions.messages = addCacheControl(requestOptions.messages);+ }+logger.debug('[AnthropicClient]', { ...requestOptions });const handleChunk = (currentChunk) => {@@ -639,6 +771,11 @@ class AnthropicClient extends BaseClient {for await (const completion of response) {// Handle each completion as before+ const type = completion?.type ?? '';+ if (tokenEventTypes.has(type)) {+ logger.debug(`[AnthropicClient] ${type}`, completion);+ this[type] = completion;+ }if (completion?.delta?.text) {handleChunk(completion.delta.text);} else if (completion.completion) {@@ -727,6 +864,8 @@ class AnthropicClient extends BaseClient {*/async titleConvo({ text, responseText = '' }) {let title = 'New Chat';+ this.message_delta = undefined;+ this.message_start = undefined;const convo = `<initial_message>${truncateText(text)}</initial_message>diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.jsindex 1a2c6aadc2..22940bae5f 100644--- a/api/app/clients/BaseClient.js+++ b/api/app/clients/BaseClient.js@@ -54,10 +54,22 @@ class BaseClient {throw new Error('Subclasses attempted to call summarizeMessages without implementing it');}- async getTokenCountForResponse(response) {- logger.debug('`[BaseClient] recordTokenUsage` not implemented.', response);+ /**+ * Abstract method to get the token count for a message. Subclasses must implement this method.+ * @param {TMessage} responseMessage+ * @returns {number}+ */+ getTokenCountForResponse(responseMessage) {+ logger.debug('`[BaseClient] recordTokenUsage` not implemented.', responseMessage);}+ /**+ * Abstract method to record token usage. Subclasses must implement this method.+ * If a correction to the token usage is needed, the method should return an object with the corrected token counts.+ * @param {number} promptTokens+ * @param {number} completionTokens+ * @returns {Promise<void>}+ */async recordTokenUsage({ promptTokens, completionTokens }) {logger.debug('`[BaseClient] recordTokenUsage` not implemented.', {promptTokens,@@ -536,13 +548,31 @@ class BaseClient {this.getTokenCountForResponse &&this.getTokenCount) {- responseMessage.tokenCount = this.getTokenCountForResponse(responseMessage);- const completionTokens = this.getTokenCount(completion);- await this.recordTokenUsage({ promptTokens, completionTokens });+ let completionTokens;++ /**+ * Metadata about input/output costs for the current message. The client+ * should provide a function to get the current stream usage metadata; if not,+ * use the legacy token estimations.+ * @type {StreamUsage | null} */+ const usage = this.getStreamUsage != null ? this.getStreamUsage() : null;++ if (usage != null && Number(usage.output_tokens) > 0) {+ responseMessage.tokenCount = usage.output_tokens;+ completionTokens = responseMessage.tokenCount;+ await this.updateUserMessageTokenCount({ usage, tokenCountMap, userMessage, opts });+ } else {+ responseMessage.tokenCount = this.getTokenCountForResponse(responseMessage);+ completionTokens = this.getTokenCount(completion);+ }++ await this.recordTokenUsage({ promptTokens, completionTokens, usage });}+if (this.userMessagePromise) {await this.userMessagePromise;}+this.responsePromise = this.saveMessageToDatabase(responseMessage, saveOptions, user);const messageCache = getLogStores(CacheKeys.MESSAGES);messageCache.set(@@ -557,6 +587,66 @@ class BaseClient {return responseMessage;}+ /**+ * Stream usage should only be used for user message token count re-calculation if:+ * - The stream usage is available, with input tokens greater than 0,+ * - the client provides a function to calculate the current token count,+ * - files are being resent with every message (default behavior; or if `false`, with no attachments),+ * - the `promptPrefix` (custom instructions) is not set.+ *+ * In these cases, the legacy token estimations would be more accurate.+ *+ * TODO: included system messages in the `orderedMessages` accounting, potentially as a+ * separate message in the UI. ChatGPT does this through "hidden" system messages.+ * @param {object} params+ * @param {StreamUsage} params.usage+ * @param {Record<string, number>} params.tokenCountMap+ * @param {TMessage} params.userMessage+ * @param {object} params.opts+ */+ async updateUserMessageTokenCount({ usage, tokenCountMap, userMessage, opts }) {+ /** @type {boolean} */+ const shouldUpdateCount =+ this.calculateCurrentTokenCount != null &&+ Number(usage.input_tokens) > 0 &&+ (this.options.resendFiles ||+ (!this.options.resendFiles && !this.options.attachments?.length)) &&+ !this.options.promptPrefix;++ if (!shouldUpdateCount) {+ return;+ }++ const userMessageTokenCount = this.calculateCurrentTokenCount({+ currentMessageId: userMessage.messageId,+ tokenCountMap,+ usage,+ });++ if (userMessageTokenCount === userMessage.tokenCount) {+ return;+ }++ userMessage.tokenCount = userMessageTokenCount;+ /*+ Note: `AskController` saves the user message, so we update the count of its `userMessage` reference+ */+ if (typeof opts?.getReqData === 'function') {+ opts.getReqData({+ userMessage,+ });+ }+ /*+ Note: we update the user message to be sure it gets the calculated token count;+ though `AskController` saves the user message, EditController does not+ */+ await this.userMessagePromise;+ await this.updateMessageInDatabase({+ messageId: userMessage.messageId,+ tokenCount: userMessageTokenCount,+ });+ }+async loadHistory(conversationId, parentMessageId = null) {logger.debug('[BaseClient] Loading history:', { conversationId, parentMessageId });@@ -644,6 +734,10 @@ class BaseClient {return { message: savedMessage, conversation };}+ /**+ * Update a message in the database.+ * @param {Partial<TMessage>} message+ */async updateMessageInDatabase(message) {await updateMessage(this.options.req, message);}diff --git a/api/app/clients/OpenAIClient.js b/api/app/clients/OpenAIClient.jsindex ddb8b7c42d..a7f97e5a3c 100644--- a/api/app/clients/OpenAIClient.js+++ b/api/app/clients/OpenAIClient.js@@ -27,9 +27,9 @@ const {createContextHandlers,} = require('./prompts');const { encodeAndFormat } = require('~/server/services/Files/images/encode');+const { spendTokens } = require('~/models/spendTokens');const { isEnabled, sleep } = require('~/server/utils');const { handleOpenAIErrors } = require('./tools/util');-const spendTokens = require('~/models/spendTokens');const { createLLM, RunManager } = require('./llm');const ChatGPTClient = require('./ChatGPTClient');const { summaryBuffer } = require('./memory');diff --git a/api/app/clients/llm/RunManager.js b/api/app/clients/llm/RunManager.jsindex 7ab0b06b52..51abe480a9 100644--- a/api/app/clients/llm/RunManager.js+++ b/api/app/clients/llm/RunManager.js@@ -1,5 +1,5 @@const { createStartHandler } = require('~/app/clients/callbacks');-const spendTokens = require('~/models/spendTokens');+const { spendTokens } = require('~/models/spendTokens');const { logger } = require('~/config');class RunManager {diff --git a/api/app/clients/prompts/addCacheControl.js b/api/app/clients/prompts/addCacheControl.jsnew file mode 100644index 0000000000..375e08e9b6--- /dev/null+++ b/api/app/clients/prompts/addCacheControl.js@@ -0,0 +1,43 @@+/**+ * Anthropic API: Adds cache control to the appropriate user messages in the payload.+ * @param {Array<AnthropicMessage>} messages - The array of message objects.+ * @returns {Array<AnthropicMessage>} - The updated array of message objects with cache control added.+ */+function addCacheControl(messages) {+ if (!Array.isArray(messages) || messages.length < 2) {+ return messages;+ }++ const updatedMessages = [...messages];+ let userMessagesFound = 0;++ for (let i = updatedMessages.length - 1; i >= 0 && userMessagesFound < 2; i--) {+ if (updatedMessages[i].role === 'user') {+ if (typeof updatedMessages[i].content === 'string') {+ updatedMessages[i] = {+ ...updatedMessages[i],+ content: [+ {+ type: 'text',+ text: updatedMessages[i].content,+ cache_control: { type: 'ephemeral' },+ },+ ],+ };+ } else if (Array.isArray(updatedMessages[i].content)) {+ updatedMessages[i] = {+ ...updatedMessages[i],+ content: updatedMessages[i].content.map((item) => ({+ ...item,+ cache_control: { type: 'ephemeral' },+ })),+ };+ }+ userMessagesFound++;+ }+ }++ return updatedMessages;+}++module.exports = addCacheControl;diff --git a/api/app/clients/prompts/addCacheControl.spec.js b/api/app/clients/prompts/addCacheControl.spec.jsnew file mode 100644index 0000000000..d95ccaf34b--- /dev/null+++ b/api/app/clients/prompts/addCacheControl.spec.js@@ -0,0 +1,164 @@+const addCacheControl = require('./addCacheControl');++describe('addCacheControl', () => {+ test('should add cache control to the last two user messages with array content', () => {+ const messages = [+ { role: 'user', content: [{ type: 'text', text: 'Hello' }] },+ { role: 'assistant', content: [{ type: 'text', text: 'Hi there' }] },+ { role: 'user', content: [{ type: 'text', text: 'How are you?' }] },+ { role: 'assistant', content: [{ type: 'text', text: 'I\'m doing well, thanks!' }] },+ { role: 'user', content: [{ type: 'text', text: 'Great!' }] },+ ];++ const result = addCacheControl(messages);++ expect(result[0].content[0]).not.toHaveProperty('cache_control');+ expect(result[2].content[0].cache_control).toEqual({ type: 'ephemeral' });+ expect(result[4].content[0].cache_control).toEqual({ type: 'ephemeral' });+ });++ test('should add cache control to the last two user messages with string content', () => {+ const messages = [+ { role: 'user', content: 'Hello' },+ { role: 'assistant', content: 'Hi there' },+ { role: 'user', content: 'How are you?' },+ { role: 'assistant', content: 'I\'m doing well, thanks!' },+ { role: 'user', content: 'Great!' },+ ];++ const result = addCacheControl(messages);++ expect(result[0].content).toBe('Hello');+ expect(result[2].content[0]).toEqual({+ type: 'text',+ text: 'How are you?',+ cache_control: { type: 'ephemeral' },+ });+ expect(result[4].content[0]).toEqual({+ type: 'text',+ text: 'Great!',+ cache_control: { type: 'ephemeral' },+ });+ });++ test('should handle mixed string and array content', () => {+ const messages = [+ { role: 'user', content: 'Hello' },+ { role: 'assistant', content: 'Hi there' },+ { role: 'user', content: [{ type: 'text', text: 'How are you?' }] },+ ];++ const result = addCacheControl(messages);++ expect(result[0].content[0]).toEqual({+ type: 'text',+ text: 'Hello',+ cache_control: { type: 'ephemeral' },+ });+ expect(result[2].content[0].cache_control).toEqual({ type: 'ephemeral' });+ });++ test('should handle less than two user messages', () => {+ const messages = [+ { role: 'user', content: 'Hello' },+ { role: 'assistant', content: 'Hi there' },+ ];++ const result = addCacheControl(messages);++ expect(result[0].content[0]).toEqual({+ type: 'text',+ text: 'Hello',+ cache_control: { type: 'ephemeral' },+ });+ expect(result[1].content).toBe('Hi there');+ });++ test('should return original array if no user messages', () => {+ const messages = [+ { role: 'assistant', content: 'Hi there' },+ { role: 'assistant', content: 'How can I help?' },+ ];++ const result = addCacheControl(messages);++ expect(result).toEqual(messages);+ });++ test('should handle empty array', () => {+ const messages = [];+ const result = addCacheControl(messages);+ expect(result).toEqual([]);+ });++ test('should handle non-array input', () => {+ const messages = 'not an array';+ const result = addCacheControl(messages);+ expect(result).toBe('not an array');+ });++ test('should not modify assistant messages', () => {+ const messages = [+ { role: 'user', content: 'Hello' },+ { role: 'assistant', content: 'Hi there' },+ { role: 'user', content: 'How are you?' },+ ];++ const result = addCacheControl(messages);++ expect(result[1].content).toBe('Hi there');+ });++ test('should handle multiple content items in user messages', () => {+ const messages = [+ {+ role: 'user',+ content: [+ { type: 'text', text: 'Hello' },+ { type: 'image', url: 'http://example.com/image.jpg' },+ ],+ },+ { role: 'assistant', content: 'Hi there' },+ { role: 'user', content: 'How are you?' },+ ];++ const result = addCacheControl(messages);++ expect(result[0].content[0].cache_control).toEqual({ type: 'ephemeral' });+ expect(result[0].content[1].cache_control).toEqual({ type: 'ephemeral' });+ expect(result[2].content[0]).toEqual({+ type: 'text',+ text: 'How are you?',+ cache_control: { type: 'ephemeral' },+ });+ });++ test('should handle an array with mixed content types', () => {+ const messages = [+ { role: 'user', content: 'Hello' },+ { role: 'assistant', content: 'Hi there' },+ { role: 'user', content: [{ type: 'text', text: 'How are you?' }] },+ { role: 'assistant', content: 'I\'m doing well, thanks!' },+ { role: 'user', content: 'Great!' },+ ];++ const result = addCacheControl(messages);+ console.dir(result, { depth: null });++ expect(result[0].content).toEqual('Hello');+ expect(result[2].content[0]).toEqual({+ type: 'text',+ text: 'How are you?',+ cache_control: { type: 'ephemeral' },+ });+ expect(result[4].content).toEqual([+ {+ type: 'text',+ text: 'Great!',+ cache_control: { type: 'ephemeral' },+ },+ ]);+ expect(result[1].content).toBe('Hi there');+ expect(result[3].content).toBe('I\'m doing well, thanks!');+ });+});diff --git a/api/app/clients/prompts/index.js b/api/app/clients/prompts/index.jsindex 9477fb30ca..364ad34b5e 100644--- a/api/app/clients/prompts/index.js+++ b/api/app/clients/prompts/index.js@@ -1,3 +1,4 @@+const addCacheControl = require('./addCacheControl');const formatMessages = require('./formatMessages');const summaryPrompts = require('./summaryPrompts');const handleInputs = require('./handleInputs');@@ -8,6 +9,7 @@ const createVisionPrompt = require('./createVisionPrompt');const createContextHandlers = require('./createContextHandlers');module.exports = {+ addCacheControl,...formatMessages,...summaryPrompts,...handleInputs,diff --git a/api/app/clients/specs/AnthropicClient.test.js b/api/app/clients/specs/AnthropicClient.test.jsindex f559159bf0..09ae9a3995 100644--- a/api/app/clients/specs/AnthropicClient.test.js+++ b/api/app/clients/specs/AnthropicClient.test.js@@ -211,7 +211,21 @@ describe('AnthropicClient', () => {expect(anthropicClient._options.defaultHeaders).toBeDefined();expect(anthropicClient._options.defaultHeaders).toHaveProperty('anthropic-beta');expect(anthropicClient._options.defaultHeaders['anthropic-beta']).toBe(- 'max-tokens-3-5-sonnet-2024-07-15',+ 'max-tokens-3-5-sonnet-2024-07-15,prompt-caching-2024-07-31',+ );+ });++ it('should add beta header for claude-3-haiku model', () => {+ const client = new AnthropicClient('test-api-key');+ const modelOptions = {+ model: 'claude-3-haiku-2028',+ };+ client.setOptions({ modelOptions });+ const anthropicClient = client.getClient(modelOptions);+ expect(anthropicClient._options.defaultHeaders).toBeDefined();+ expect(anthropicClient._options.defaultHeaders).toHaveProperty('anthropic-beta');+ expect(anthropicClient._options.defaultHeaders['anthropic-beta']).toBe(+ 'prompt-caching-2024-07-31',);});@@ -226,4 +240,145 @@ describe('AnthropicClient', () => {expect(anthropicClient.defaultHeaders).not.toHaveProperty('anthropic-beta');});});++ describe('calculateCurrentTokenCount', () => {+ let client;++ beforeEach(() => {+ client = new AnthropicClient('test-api-key');+ });++ it('should calculate correct token count when usage is provided', () => {+ const tokenCountMap = {+ msg1: 10,+ msg2: 20,+ currentMsg: 30,+ };+ const currentMessageId = 'currentMsg';+ const usage = {+ input_tokens: 70,+ output_tokens: 50,+ };++ const result = client.calculateCurrentTokenCount({ tokenCountMap, currentMessageId, usage });++ expect(result).toBe(40); // 70 - (10 + 20) = 40+ });++ it('should return original estimate if calculation results in negative value', () => {+ const tokenCountMap = {+ msg1: 40,+ msg2: 50,+ currentMsg: 30,+ };+ const currentMessageId = 'currentMsg';+ const usage = {+ input_tokens: 80,+ output_tokens: 50,+ };++ const result = client.calculateCurrentTokenCount({ tokenCountMap, currentMessageId, usage });++ expect(result).toBe(30); // Original estimate+ });++ it('should handle cache creation and read input tokens', () => {+ const tokenCountMap = {+ msg1: 10,+ msg2: 20,+ currentMsg: 30,+ };+ const currentMessageId = 'currentMsg';+ const usage = {+ input_tokens: 50,+ cache_creation_input_tokens: 10,+ cache_read_input_tokens: 20,+ output_tokens: 40,+ };++ const result = client.calculateCurrentTokenCount({ tokenCountMap, currentMessageId, usage });++ expect(result).toBe(50); // (50 + 10 + 20) - (10 + 20) = 50+ });++ it('should handle missing usage properties', () => {+ const tokenCountMap = {+ msg1: 10,+ msg2: 20,+ currentMsg: 30,+ };+ const currentMessageId = 'currentMsg';+ const usage = {+ output_tokens: 40,+ };++ const result = client.calculateCurrentTokenCount({ tokenCountMap, currentMessageId, usage });++ expect(result).toBe(30); // Original estimate+ });++ it('should handle empty tokenCountMap', () => {+ const tokenCountMap = {};+ const currentMessageId = 'currentMsg';+ const usage = {+ input_tokens: 50,+ output_tokens: 40,+ };++ const result = client.calculateCurrentTokenCount({ tokenCountMap, currentMessageId, usage });++ expect(result).toBe(50);+ expect(Number.isNaN(result)).toBe(false);+ });++ it('should handle zero values in usage', () => {+ const tokenCountMap = {+ msg1: 10,+ currentMsg: 20,+ };+ const currentMessageId = 'currentMsg';+ const usage = {+ input_tokens: 0,+ cache_creation_input_tokens: 0,+ cache_read_input_tokens: 0,+ output_tokens: 0,+ };++ const result = client.calculateCurrentTokenCount({ tokenCountMap, currentMessageId, usage });++ expect(result).toBe(20); // Should return original estimate+ expect(Number.isNaN(result)).toBe(false);+ });++ it('should handle undefined usage', () => {+ const tokenCountMap = {+ msg1: 10,+ currentMsg: 20,+ };+ const currentMessageId = 'currentMsg';+ const usage = undefined;++ const result = client.calculateCurrentTokenCount({ tokenCountMap, currentMessageId, usage });++ expect(result).toBe(20); // Should return original estimate+ expect(Number.isNaN(result)).toBe(false);+ });++ it('should handle non-numeric values in tokenCountMap', () => {+ const tokenCountMap = {+ msg1: 'ten',+ currentMsg: 20,+ };+ const currentMessageId = 'currentMsg';+ const usage = {+ input_tokens: 30,+ output_tokens: 10,+ };++ const result = client.calculateCurrentTokenCount({ tokenCountMap, currentMessageId, usage });++ expect(result).toBe(30); // Should return 30 (input_tokens) - 0 (ignored 'ten') = 30+ expect(Number.isNaN(result)).toBe(false);+ });+ });});diff --git a/api/models/Message.js b/api/models/Message.jsindex 0890064ca4..848570c4e3 100644--- a/api/models/Message.js+++ b/api/models/Message.js@@ -212,8 +212,8 @@ async function updateMessageText(req, { messageId, text }) {** @async* @function updateMessage- * @param {Object} message - The message object containing update data.* @param {Object} req - The request object.+ * @param {Object} message - The message object containing update data.* @param {string} message.messageId - The unique identifier for the message.* @param {string} [message.text] - The new text content of the message.* @param {Object[]} [message.files] - The files associated with the message.diff --git a/api/models/Transaction.js b/api/models/Transaction.jsindex 0d11ab5374..3ee4af8b10 100644--- a/api/models/Transaction.js+++ b/api/models/Transaction.js@@ -1,12 +1,12 @@const mongoose = require('mongoose');const { isEnabled } = require('../server/utils/handleText');const transactionSchema = require('./schema/transaction');-const { getMultiplier } = require('./tx');+const { getMultiplier, getCacheMultiplier } = require('./tx');const { logger } = require('~/config');const Balance = require('./Balance');const cancelRate = 1.15;-// Method to calculate and set the tokenValue for a transaction+/** Method to calculate and set the tokenValue for a transaction */transactionSchema.methods.calculateTokenValue = function () {if (!this.valueKey || !this.tokenType) {this.tokenValue = this.rawAmount;@@ -21,15 +21,17 @@ transactionSchema.methods.calculateTokenValue = function () {}};-// Static method to create a transaction and update the balance-transactionSchema.statics.create = async function (transactionData) {+/**+ * Static method to create a transaction and update the balance+ * @param {txData} txData - Transaction data.+ */+transactionSchema.statics.create = async function (txData) {const Transaction = this;- const transaction = new Transaction(transactionData);- transaction.endpointTokenConfig = transactionData.endpointTokenConfig;+ const transaction = new Transaction(txData);+ transaction.endpointTokenConfig = txData.endpointTokenConfig;transaction.calculateTokenValue();- // Save the transactionawait transaction.save();if (!isEnabled(process.env.CHECK_BALANCE)) {@@ -57,6 +59,104 @@ transactionSchema.statics.create = async function (transactionData) {};};+/**+ * Static method to create a structured transaction and update the balance+ * @param {txData} txData - Transaction data.+ */+transactionSchema.statics.createStructured = async function (txData) {+ const Transaction = this;++ const transaction = new Transaction({+ ...txData,+ endpointTokenConfig: txData.endpointTokenConfig,+ });++ transaction.calculateStructuredTokenValue();++ await transaction.save();++ if (!isEnabled(process.env.CHECK_BALANCE)) {+ return transaction;+ }++ let balance = await Balance.findOne({ user: transaction.user }).lean();+ let incrementValue = transaction.tokenValue;++ if (balance && balance?.tokenCredits + incrementValue < 0) {+ incrementValue = -balance.tokenCredits;+ }++ balance = await Balance.findOneAndUpdate(+ { user: transaction.user },+ { $inc: { tokenCredits: incrementValue } },+ { upsert: true, new: true },+ ).lean();++ return {+ rate: transaction.rate,+ user: transaction.user.toString(),+ balance: balance.tokenCredits,+ [transaction.tokenType]: incrementValue,+ };+};++/** Method to calculate token value for structured tokens */+transactionSchema.methods.calculateStructuredTokenValue = function () {+ if (!this.tokenType) {+ this.tokenValue = this.rawAmount;+ return;+ }++ const { model, endpointTokenConfig } = this;++ if (this.tokenType === 'prompt') {+ const inputMultiplier = getMultiplier({ tokenType: 'prompt', model, endpointTokenConfig });+ const writeMultiplier =+ getCacheMultiplier({ cacheType: 'write', model, endpointTokenConfig }) ?? inputMultiplier;+ const readMultiplier =+ getCacheMultiplier({ cacheType: 'read', model, endpointTokenConfig }) ?? inputMultiplier;++ this.rateDetail = {+ input: inputMultiplier,+ write: writeMultiplier,+ read: readMultiplier,+ };++ const totalTokens = (this.inputTokens || 0) + (this.writeTokens || 0) + (this.readTokens || 0);++ if (totalTokens > 0) {+ this.rate =+ (inputMultiplier * (this.inputTokens || 0) ++ writeMultiplier * (this.writeTokens || 0) ++ readMultiplier * (this.readTokens || 0)) /+ totalTokens;+ } else {+ this.rate = inputMultiplier; // Default to input rate if no tokens+ }++ this.tokenValue =+ this.inputTokens * inputMultiplier ++ (this.writeTokens || 0) * writeMultiplier ++ (this.readTokens || 0) * readMultiplier;+ } else {+ const multiplier = Math.abs(+ getMultiplier({ tokenType: this.tokenType, model, endpointTokenConfig }),+ );+ this.rate = multiplier;+ this.tokenValue = this.rawAmount * multiplier;+ }++ if (this.context && this.tokenType === 'completion' && this.context === 'incomplete') {+ this.tokenValue = Math.ceil(this.tokenValue * cancelRate);+ this.rate *= cancelRate;+ if (this.rateDetail) {+ this.rateDetail = Object.fromEntries(+ Object.entries(this.rateDetail).map(([k, v]) => [k, v * cancelRate]),+ );+ }+ }+};+const Transaction = mongoose.model('Transaction', transactionSchema);/**diff --git a/api/models/schema/transaction.js b/api/models/schema/transaction.jsindex 50de734805..8cb9ba59cc 100644--- a/api/models/schema/transaction.js+++ b/api/models/schema/transaction.js@@ -30,6 +30,9 @@ const transactionSchema = mongoose.Schema(rate: Number,rawAmount: Number,tokenValue: Number,+ inputTokens: { type: Number },+ writeTokens: { type: Number },+ readTokens: { type: Number },},{timestamps: true,diff --git a/api/models/spendTokens.js b/api/models/spendTokens.jsindex 917d0c93db..12b9389958 100644--- a/api/models/spendTokens.js+++ b/api/models/spendTokens.js@@ -11,7 +11,7 @@ const { logger } = require('~/config');* @param {String} txData.conversationId - The ID of the conversation.* @param {String} txData.model - The model name.* @param {String} txData.context - The context in which the transaction is made.- * @param {String} [txData.endpointTokenConfig] - The current endpoint token config.+ * @param {EndpointTokenConfig} [txData.endpointTokenConfig] - The current endpoint token config.* @param {String} [txData.valueKey] - The value key (optional).* @param {Object} tokenUsage - The number of tokens used.* @param {Number} tokenUsage.promptTokens - The number of prompt tokens used.@@ -66,4 +66,74 @@ const spendTokens = async (txData, tokenUsage) => {}};-module.exports = spendTokens;+/**+ * Creates transactions to record the spending of structured tokens.+ *+ * @function+ * @async+ * @param {Object} txData - Transaction data.+ * @param {mongoose.Schema.Types.ObjectId} txData.user - The user ID.+ * @param {String} txData.conversationId - The ID of the conversation.+ * @param {String} txData.model - The model name.+ * @param {String} txData.context - The context in which the transaction is made.+ * @param {EndpointTokenConfig} [txData.endpointTokenConfig] - The current endpoint token config.+ * @param {String} [txData.valueKey] - The value key (optional).+ * @param {Object} tokenUsage - The number of tokens used.+ * @param {Object} tokenUsage.promptTokens - The number of prompt tokens used.+ * @param {Number} tokenUsage.promptTokens.input - The number of input tokens.+ * @param {Number} tokenUsage.promptTokens.write - The number of write tokens.+ * @param {Number} tokenUsage.promptTokens.read - The number of read tokens.+ * @param {Number} tokenUsage.completionTokens - The number of completion tokens used.+ * @returns {Promise<void>} - Returns nothing.+ * @throws {Error} - Throws an error if there's an issue creating the transactions.+ */+const spendStructuredTokens = async (txData, tokenUsage) => {+ const { promptTokens, completionTokens } = tokenUsage;+ logger.debug(+ `[spendStructuredTokens] conversationId: ${txData.conversationId}${+ txData?.context ? ` | Context: ${txData?.context}` : ''+ } | Token usage: `,+ {+ promptTokens,+ completionTokens,+ },+ );+ let prompt, completion;+ try {+ if (promptTokens) {+ const { input = 0, write = 0, read = 0 } = promptTokens;+ const promptAmount = input + write + read;+ prompt = await Transaction.createStructured({+ ...txData,+ tokenType: 'prompt',+ rawAmount: -promptAmount,+ inputTokens: input,+ writeTokens: write,+ readTokens: read,+ });+ }++ if (completionTokens) {+ completion = await Transaction.create({+ ...txData,+ tokenType: 'completion',+ rawAmount: -completionTokens,+ });+ }++ prompt &&+ completion &&+ logger.debug('[spendStructuredTokens] Transaction data record against balance:', {+ user: txData.user,+ prompt: prompt.tokenValue,+ promptRate: prompt.rate,+ completion: completion.tokenValue,+ completionRate: completion.rate,+ balance: completion.balance,+ });+ } catch (err) {+ logger.error('[spendStructuredTokens]', err);+ }+};++module.exports = { spendTokens, spendStructuredTokens };diff --git a/api/models/tx.js b/api/models/tx.jsindex 3018de4174..adbedd82c3 100644--- a/api/models/tx.js+++ b/api/models/tx.js@@ -70,6 +70,17 @@ const tokenValues = Object.assign(bedrockValues,);+/**+ * Mapping of model token sizes to their respective multipliers for cached input, read and write.+ * See Anthropic's documentation on this: https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching#pricing+ * The rates are 1 USD per 1M tokens.+ * @type {Object.<string, {write: number, read: number }>}+ */+const cacheTokenValues = {+ 'claude-3-5-sonnet': { write: 3.75, read: 0.3 },+ 'claude-3-haiku': { write: 0.3, read: 0.03 },+};+/*** Retrieves the key associated with a given model name.*@@ -122,7 +133,7 @@ const getValueKey = (model, endpoint) => {** @param {Object} params - The parameters for the function.* @param {string} [params.valueKey] - The key corresponding to the model name.- * @param {string} [params.tokenType] - The type of token (e.g., 'prompt' or 'completion').+ * @param {'prompt' | 'completion'} [params.tokenType] - The type of token (e.g., 'prompt' or 'completion').* @param {string} [params.model] - The model name to derive the value key from if not provided.* @param {string} [params.endpoint] - The endpoint name to derive the value key from if not provided.* @param {EndpointTokenConfig} [params.endpointTokenConfig] - The token configuration for the endpoint.@@ -147,7 +158,41 @@ const getMultiplier = ({ valueKey, tokenType, model, endpoint, endpointTokenConf}// If we got this far, and values[tokenType] is undefined somehow, return a rough average of default multipliers- return tokenValues[valueKey][tokenType] ?? defaultRate;+ return tokenValues[valueKey]?.[tokenType] ?? defaultRate;+};++/**+ * Retrieves the cache multiplier for a given value key and token type. If no value key is provided,+ * it attempts to derive it from the model name.+ *+ * @param {Object} params - The parameters for the function.+ * @param {string} [params.valueKey] - The key corresponding to the model name.+ * @param {'write' | 'read'} [params.cacheType] - The type of token (e.g., 'write' or 'read').+ * @param {string} [params.model] - The model name to derive the value key from if not provided.+ * @param {string} [params.endpoint] - The endpoint name to derive the value key from if not provided.+ * @param {EndpointTokenConfig} [params.endpointTokenConfig] - The token configuration for the endpoint.+ * @returns {number | null} The multiplier for the given parameters, or `null` if not found.+ */+const getCacheMultiplier = ({ valueKey, cacheType, model, endpoint, endpointTokenConfig }) => {+ if (endpointTokenConfig) {+ return endpointTokenConfig?.[model]?.[cacheType] ?? null;+ }++ if (valueKey && cacheType) {+ return cacheTokenValues[valueKey]?.[cacheType] ?? null;+ }++ if (!cacheType || !model) {+ return null;+ }++ valueKey = getValueKey(model, endpoint);+ if (!valueKey) {+ return null;+ }++ // If we got this far, and values[cacheType] is undefined somehow, return a rough average of default multipliers+ return cacheTokenValues[valueKey]?.[cacheType] ?? null;};-module.exports = { tokenValues, getValueKey, getMultiplier, defaultRate };+module.exports = { tokenValues, getValueKey, getMultiplier, getCacheMultiplier, defaultRate };diff --git a/api/models/tx.spec.js b/api/models/tx.spec.jsindex 8226da5d9b..db0251dbb6 100644--- a/api/models/tx.spec.js+++ b/api/models/tx.spec.js@@ -1,4 +1,10 @@-const { getValueKey, getMultiplier, defaultRate, tokenValues } = require('./tx');+const {+ defaultRate,+ tokenValues,+ getValueKey,+ getMultiplier,+ getCacheMultiplier,+} = require('./tx');describe('getValueKey', () => {it('should return "16k" for model name containing "gpt-3.5-turbo-16k"', () => {@@ -243,3 +249,76 @@ describe('AWS Bedrock Model Tests', () => {expect(results.every(Boolean)).toBe(true);});});++describe('getCacheMultiplier', () => {+ it('should return the correct cache multiplier for a given valueKey and cacheType', () => {+ expect(getCacheMultiplier({ valueKey: 'claude-3-5-sonnet', cacheType: 'write' })).toBe(3.75);+ expect(getCacheMultiplier({ valueKey: 'claude-3-5-sonnet', cacheType: 'read' })).toBe(0.3);+ expect(getCacheMultiplier({ valueKey: 'claude-3-haiku', cacheType: 'write' })).toBe(0.3);+ expect(getCacheMultiplier({ valueKey: 'claude-3-haiku', cacheType: 'read' })).toBe(0.03);+ });++ it('should return null if cacheType is provided but not found in cacheTokenValues', () => {+ expect(+ getCacheMultiplier({ valueKey: 'claude-3-5-sonnet', cacheType: 'unknownType' }),+ ).toBeNull();+ });++ it('should derive the valueKey from the model if not provided', () => {+ expect(getCacheMultiplier({ cacheType: 'write', model: 'claude-3-5-sonnet-20240620' })).toBe(+ 3.75,+ );+ expect(getCacheMultiplier({ cacheType: 'read', model: 'claude-3-haiku-20240307' })).toBe(0.03);+ });++ it('should return null if only model or cacheType is missing', () => {+ expect(getCacheMultiplier({ cacheType: 'write' })).toBeNull();+ expect(getCacheMultiplier({ model: 'claude-3-5-sonnet' })).toBeNull();+ });++ it('should return null if derived valueKey does not match any known patterns', () => {+ expect(getCacheMultiplier({ cacheType: 'write', model: 'gpt-4-some-other-info' })).toBeNull();+ });++ it('should handle endpointTokenConfig if provided', () => {+ const endpointTokenConfig = {+ 'custom-model': {+ write: 5,+ read: 1,+ },+ };+ expect(+ getCacheMultiplier({ model: 'custom-model', cacheType: 'write', endpointTokenConfig }),+ ).toBe(5);+ expect(+ getCacheMultiplier({ model: 'custom-model', cacheType: 'read', endpointTokenConfig }),+ ).toBe(1);+ });++ it('should return null if model is not found in endpointTokenConfig', () => {+ const endpointTokenConfig = {+ 'custom-model': {+ write: 5,+ read: 1,+ },+ };+ expect(+ getCacheMultiplier({ model: 'unknown-model', cacheType: 'write', endpointTokenConfig }),+ ).toBeNull();+ });++ it('should handle models with "bedrock/" prefix', () => {+ expect(+ getCacheMultiplier({+ model: 'bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0',+ cacheType: 'write',+ }),+ ).toBe(3.75);+ expect(+ getCacheMultiplier({+ model: 'bedrock/anthropic.claude-3-haiku-20240307-v1:0',+ cacheType: 'read',+ }),+ ).toBe(0.03);+ });+});diff --git a/api/server/middleware/abortMiddleware.js b/api/server/middleware/abortMiddleware.jsindex 71af96c532..e855c0cb69 100644--- a/api/server/middleware/abortMiddleware.js+++ b/api/server/middleware/abortMiddleware.js@@ -2,9 +2,9 @@ const { isAssistantsEndpoint } = require('librechat-data-provider');const { sendMessage, sendError, countTokens, isEnabled } = require('~/server/utils');const { truncateText, smartTruncateText } = require('~/app/clients/prompts');const clearPendingReq = require('~/cache/clearPendingReq');+const { spendTokens } = require('~/models/spendTokens');const abortControllers = require('./abortControllers');const { saveMessage, getConvo } = require('~/models');-const spendTokens = require('~/models/spendTokens');const { abortRun } = require('./abortRun');const { logger } = require('~/config');diff --git a/api/server/services/Threads/manage.js b/api/server/services/Threads/manage.jsindex 920e68af55..5155d61109 100644--- a/api/server/services/Threads/manage.js+++ b/api/server/services/Threads/manage.js@@ -8,8 +8,8 @@ const {} = require('librechat-data-provider');const { retrieveAndProcessFile } = require('~/server/services/Files/process');const { recordMessage, getMessages } = require('~/models/Message');+const { spendTokens } = require('~/models/spendTokens');const { saveConvo } = require('~/models/Conversation');-const spendTokens = require('~/models/spendTokens');const { countTokens } = require('~/server/utils');/**diff --git a/api/typedefs.js b/api/typedefs.jsindex c8f46c6d9b..f69f7b1843 100644--- a/api/typedefs.js+++ b/api/typedefs.js@@ -26,6 +26,24 @@* @memberof typedefs*/+/**+ * @exports AnthropicMessage+ * @typedef {import('@anthropic-ai/sdk').default.MessageParam} AnthropicMessage+ * @memberof typedefs+ */++/**+ * @exports AnthropicMessageStartEvent+ * @typedef {import('@anthropic-ai/sdk').default.MessageStartEvent} AnthropicMessageStartEvent+ * @memberof typedefs+ */++/**+ * @exports AnthropicMessageDeltaEvent+ * @typedef {import('@anthropic-ai/sdk').default.MessageDeltaEvent} AnthropicMessageDeltaEvent+ * @memberof typedefs+ */+/*** @exports GenerativeModel* @typedef {import('@google/generative-ai').GenerativeModel} GenerativeModel@@ -1311,6 +1329,33 @@* @method messageCompleted Handles the completion of a message processing.*/+/* TX Types */++/**+ * @typedef {object} txData - Transaction data.+ * @property {mongoose.Schema.Types.ObjectId} user - The user ID.+ * @property {String} conversationId - The ID of the conversation.+ * @property {String} model - The model name.+ * @property {String} context - The context in which the transaction is made.+ * @property {EndpointTokenConfig} [endpointTokenConfig] - The current endpoint token config.+ * @property {object} [cacheUsage] - Cache usage, if any.+ * @property {String} [valueKey] - The value key (optional).+ * @memberof typedefs+ */++/**+ * https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching#pricing+ * @typedef {object} AnthropicStreamUsage - Stream usage for Anthropic+ * @property {number} [input_tokens] - The number of input tokens used.+ * @property {number} [cache_creation_input_tokens] - The number of cache creation input tokens used (write).+ * @property {number} [cache_read_input_tokens] - The number of cache input tokens used (read).+ * @property {number} [output_tokens] - The number of output tokens used.+ */++/**+ * @typedef {AnthropicStreamUsage} StreamUsage - Stream usage for all providers (currently only Anthropic)+ */+/* Native app/client methods *//**