Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

fix(OpenAIClient): use official SDK to identify client and avoid false Rate Limit Error (#1161)

Danny Avila5ab9802aa912
diff --git a/.eslintrc.js b/.eslintrc.js
index 9e7858375e..53277e02c7 100644
--- a/.eslintrc.js
+++ b/.eslintrc.js
@@ -61,6 +61,7 @@ module.exports = {
'no-restricted-syntax': 'off',
'react/prop-types': ['off'],
'react/display-name': ['off'],
+ 'no-unused-vars': ['error', { varsIgnorePattern: '^_' }],
quotes: ['error', 'single'],
},
overrides: [
diff --git a/api/app/clients/OpenAIClient.js b/api/app/clients/OpenAIClient.js
index ccb3646e8f..b16c070dc1 100644
--- a/api/app/clients/OpenAIClient.js
+++ b/api/app/clients/OpenAIClient.js
@@ -1,15 +1,17 @@
-const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken');
+const OpenAI = require('openai');
const { HttpsProxyAgent } = require('https-proxy-agent');
-const ChatGPTClient = require('./ChatGPTClient');
-const BaseClient = require('./BaseClient');
-const { getModelMaxTokens, genAzureChatCompletion } = require('../../utils');
+const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken');
+const { getModelMaxTokens, genAzureChatCompletion, extractBaseURL } = require('../../utils');
const { truncateText, formatMessage, CUT_OFF_PROMPT } = require('./prompts');
const spendTokens = require('../../models/spendTokens');
+const { handleOpenAIErrors } = require('./tools/util');
const { isEnabled } = require('../../server/utils');
const { createLLM, RunManager } = require('./llm');
+const ChatGPTClient = require('./ChatGPTClient');
const { summaryBuffer } = require('./memory');
const { runTitleChain } = require('./chains');
const { tokenSplit } = require('./document');
+const BaseClient = require('./BaseClient');
// Cache to store Tiktoken instances
const tokenizersCache = {};
@@ -74,7 +76,7 @@ class OpenAIClient extends BaseClient {
}
const { OPENROUTER_API_KEY, OPENAI_FORCE_PROMPT } = process.env ?? {};
- if (OPENROUTER_API_KEY) {
+ if (OPENROUTER_API_KEY && !this.azure) {
this.apiKey = OPENROUTER_API_KEY;
this.useOpenRouter = true;
}
@@ -88,7 +90,11 @@ class OpenAIClient extends BaseClient {
this.isChatCompletion = this.useOpenRouter || !!reverseProxy || model.includes('gpt-');
this.isChatGptModel = this.isChatCompletion;
- if (model.includes('text-davinci-003') || model.includes('instruct') || this.FORCE_PROMPT) {
+ if (
+ model.includes('text-davinci') ||
+ model.includes('gpt-3.5-turbo-instruct') ||
+ this.FORCE_PROMPT
+ ) {
this.isChatCompletion = false;
this.isChatGptModel = false;
}
@@ -134,7 +140,7 @@ class OpenAIClient extends BaseClient {
if (reverseProxy) {
this.completionsUrl = reverseProxy;
- this.langchainProxy = reverseProxy.match(/.*v1/)?.[0];
+ this.langchainProxy = extractBaseURL(reverseProxy);
!this.langchainProxy &&
console.warn(`The reverse proxy URL ${reverseProxy} is not valid for Plugins.
The url must follow OpenAI specs, for example: https://localhost:8080/v1/chat/completions
@@ -356,7 +362,9 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
let result = null;
let streamResult = null;
this.modelOptions.user = this.user;
- if (typeof opts.onProgress === 'function') {
+ const invalidBaseUrl = this.completionsUrl && extractBaseURL(this.completionsUrl) === null;
+ const useOldMethod = !!(this.azure || invalidBaseUrl);
+ if (typeof opts.onProgress === 'function' && useOldMethod) {
await this.getCompletion(
payload,
(progressMessage) => {
@@ -399,6 +407,13 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s
},
opts.abortController || new AbortController(),
);
+ } else if (typeof opts.onProgress === 'function') {
+ reply = await this.chatCompletion({
+ payload,
+ clientOptions: opts,
+ onProgress: opts.onProgress,
+ abortController: opts.abortController,
+ });
} else {
result = await this.getCompletion(
payload,
@@ -669,6 +684,135 @@ ${convo}
content: response.text,
});
}
+
+ async chatCompletion({ payload, onProgress, clientOptions, abortController = null }) {
+ let error = null;
+ const errorCallback = (err) => (error = err);
+ let intermediateReply = '';
+ try {
+ if (!abortController) {
+ abortController = new AbortController();
+ }
+ const modelOptions = { ...this.modelOptions };
+ if (typeof onProgress === 'function') {
+ modelOptions.stream = true;
+ }
+ if (this.isChatGptModel) {
+ modelOptions.messages = payload;
+ } else {
+ modelOptions.prompt = payload;
+ }
+
+ const { debug } = this.options;
+ const url = extractBaseURL(this.completionsUrl);
+ if (debug) {
+ console.debug('baseURL', url);
+ console.debug('modelOptions', modelOptions);
+ }
+ const opts = {
+ baseURL: url,
+ };
+
+ if (this.useOpenRouter) {
+ opts.defaultHeaders = {
+ 'HTTP-Referer': 'https://librechat.ai',
+ 'X-Title': 'LibreChat',
+ };
+ }
+
+ if (this.options.headers) {
+ opts.defaultHeaders = { ...opts.defaultHeaders, ...this.options.headers };
+ }
+
+ if (this.options.proxy) {
+ opts.httpAgent = new HttpsProxyAgent(this.options.proxy);
+ }
+
+ let chatCompletion;
+ const openai = new OpenAI({
+ apiKey: this.apiKey,
+ ...opts,
+ });
+
+ if (modelOptions.stream) {
+ const stream = await openai.beta.chat.completions
+ .stream({
+ ...modelOptions,
+ stream: true,
+ })
+ .on('abort', () => {
+ /* Do nothing here */
+ })
+ .on('error', (err) => {
+ handleOpenAIErrors(err, errorCallback, 'stream');
+ });
+
+ for await (const chunk of stream) {
+ const token = chunk.choices[0]?.delta?.content || '';
+ intermediateReply += token;
+ onProgress(token);
+ if (abortController.signal.aborted) {
+ stream.controller.abort();
+ break;
+ }
+ }
+
+ chatCompletion = await stream.finalChatCompletion().catch((err) => {
+ handleOpenAIErrors(err, errorCallback, 'finalChatCompletion');
+ });
+ }
+ // regular completion
+ else {
+ chatCompletion = await openai.chat.completions
+ .create({
+ ...modelOptions,
+ })
+ .catch((err) => {
+ handleOpenAIErrors(err, errorCallback, 'create');
+ });
+ }
+
+ if (!chatCompletion && error) {
+ throw new Error(error);
+ } else if (!chatCompletion) {
+ throw new Error('Chat completion failed');
+ }
+
+ const { message, finish_reason } = chatCompletion.choices[0];
+ if (chatCompletion && typeof clientOptions.addMetadata === 'function') {
+ clientOptions.addMetadata({ finish_reason });
+ }
+
+ return message.content;
+ } catch (err) {
+ if (
+ err?.message?.includes('abort') ||
+ (err instanceof OpenAI.APIError && err?.message?.includes('abort'))
+ ) {
+ return '';
+ }
+ if (
+ err?.message?.includes('missing finish_reason') ||
+ (err instanceof OpenAI.OpenAIError && err?.message?.includes('missing finish_reason'))
+ ) {
+ await abortController.abortCompletion();
+ return intermediateReply;
+ } else if (err instanceof OpenAI.APIError) {
+ console.log(err.name);
+ console.log(err.status);
+ console.log(err.headers);
+ if (intermediateReply) {
+ return intermediateReply;
+ } else {
+ throw err;
+ }
+ } else {
+ console.warn('[OpenAIClient.chatCompletion] Unhandled error type');
+ console.error(err);
+ throw err;
+ }
+ }
+ }
}
module.exports = OpenAIClient;
diff --git a/api/app/clients/PluginsClient.js b/api/app/clients/PluginsClient.js
index 5e9bde1435..801219c842 100644
--- a/api/app/clients/PluginsClient.js
+++ b/api/app/clients/PluginsClient.js
@@ -6,6 +6,7 @@ const { addImages, buildErrorInput, buildPromptPrefix } = require('./output_pars
const checkBalance = require('../../models/checkBalance');
const { formatLangChainMessages } = require('./prompts');
const { isEnabled } = require('../../server/utils');
+const { extractBaseURL } = require('../../utils');
const { SelfReflectionTool } = require('./tools');
const { loadTools } = require('./tools/util');
@@ -34,7 +35,7 @@ class PluginsClient extends OpenAIClient {
this.isGpt3 = this.modelOptions?.model?.includes('gpt-3');
if (this.options.reverseProxyUrl) {
- this.langchainProxy = this.options.reverseProxyUrl.match(/.*v1/)?.[0];
+ this.langchainProxy = extractBaseURL(this.options.reverseProxyUrl);
!this.langchainProxy &&
console.warn(`The reverse proxy URL ${this.options.reverseProxyUrl} is not valid for Plugins.
The url must follow OpenAI specs, for example: https://localhost:8080/v1/chat/completions
diff --git a/api/app/clients/specs/OpenAIClient.test.js b/api/app/clients/specs/OpenAIClient.test.js
index 7257aec36a..6c54a6b09c 100644
--- a/api/app/clients/specs/OpenAIClient.test.js
+++ b/api/app/clients/specs/OpenAIClient.test.js
@@ -86,7 +86,7 @@ describe('OpenAIClient', () => {
client.setOptions({ reverseProxyUrl: 'https://example.com/completions' });
expect(client.completionsUrl).toBe('https://example.com/completions');
- expect(client.langchainProxy).toBeUndefined();
+ expect(client.langchainProxy).toBe(null);
});
});
diff --git a/api/app/clients/tools/util/handleOpenAIErrors.js b/api/app/clients/tools/util/handleOpenAIErrors.js
new file mode 100644
index 0000000000..b5a31f7f40
--- /dev/null
+++ b/api/app/clients/tools/util/handleOpenAIErrors.js
@@ -0,0 +1,30 @@
+const OpenAI = require('openai');
+
+/**
+ * Handles errors that may occur when making requests to OpenAI's API.
+ * It checks the instance of the error and prints a specific warning message
+ * to the console depending on the type of error encountered.
+ * It then calls an optional error callback function with the error object.
+ *
+ * @param {Error} err - The error object thrown by OpenAI API.
+ * @param {Function} errorCallback - A callback function that is called with the error object.
+ * @param {string} [context='stream'] - A string providing context where the error occurred, defaults to 'stream'.
+ */
+async function handleOpenAIErrors(err, errorCallback, context = 'stream') {
+ if (err instanceof OpenAI.APIError && err?.message?.includes('abort')) {
+ console.warn(`[OpenAIClient.chatCompletion][${context}] Aborted Message`);
+ }
+ if (err instanceof OpenAI.OpenAIError && err?.message?.includes('missing finish_reason')) {
+ console.warn(`[OpenAIClient.chatCompletion][${context}] Missing finish_reason`);
+ } else if (err instanceof OpenAI.APIError) {
+ console.warn(`[OpenAIClient.chatCompletion][${context}] API Error`);
+ } else {
+ console.warn(`[OpenAIClient.chatCompletion][${context}] Unhandled error type`);
+ }
+
+ if (errorCallback) {
+ errorCallback(err);
+ }
+}
+
+module.exports = handleOpenAIErrors;
diff --git a/api/app/clients/tools/util/index.js b/api/app/clients/tools/util/index.js
index 9c96fb50f3..ea67bb4ced 100644
--- a/api/app/clients/tools/util/index.js
+++ b/api/app/clients/tools/util/index.js
@@ -1,6 +1,8 @@
const { validateTools, loadTools } = require('./handleTools');
+const handleOpenAIErrors = require('./handleOpenAIErrors');
module.exports = {
+ handleOpenAIErrors,
validateTools,
loadTools,
};
diff --git a/api/server/services/ModelService.js b/api/server/services/ModelService.js
index 893e348e03..4dbc780693 100644
--- a/api/server/services/ModelService.js
+++ b/api/server/services/ModelService.js
@@ -1,6 +1,7 @@
const Keyv = require('keyv');
const axios = require('axios');
const { isEnabled } = require('../utils');
+const { extractBaseURL } = require('../../utils');
const keyvRedis = require('../../cache/keyvRedis');
// const { getAzureCredentials, genAzureChatCompletion } = require('../../utils/');
const { openAIApiKey, userProvidedOpenAI } = require('./EndpointService').config;
@@ -30,7 +31,7 @@ const fetchOpenAIModels = async (opts = { azure: false, plugins: false }, _model
}
if (reverseProxyUrl) {
- basePath = reverseProxyUrl.match(/.*v1/)?.[0];
+ basePath = extractBaseURL(reverseProxyUrl);
}
const cachedModels = await modelsCache.get(basePath);
diff --git a/api/utils/extractBaseURL.js b/api/utils/extractBaseURL.js
new file mode 100644
index 0000000000..83565a4caf
--- /dev/null
+++ b/api/utils/extractBaseURL.js
@@ -0,0 +1,48 @@
+/**
+ * Extracts a valid OpenAI baseURL from a given string, matching "url/v1," also an added suffix,
+ * ending with "/openai" (to allow the Cloudflare, LiteLLM pattern).
+ *
+ * Examples:
+ * - `https://open.ai/v1/chat` -> `https://open.ai/v1`
+ * - `https://open.ai/v1/chat/completions` -> `https://open.ai/v1`
+ * - `https://open.ai/v1/ACCOUNT/GATEWAY/openai/completions` -> `https://open.ai/v1/ACCOUNT/GATEWAY/openai`
+ * - `https://open.ai/v1/hi/openai` -> `https://open.ai/v1/hi/openai`
+ *
+ * @param {string} url - The URL to be processed.
+ * @returns {string|null} The matched pattern or null if no match is found.
+ */
+function extractBaseURL(url) {
+ // First, let's make sure the URL contains '/v1'.
+ if (!url.includes('/v1')) {
+ return null;
+ }
+
+ // Find the index of '/v1' to use it as a reference point.
+ const v1Index = url.indexOf('/v1');
+
+ // Extract the part of the URL up to and including '/v1'.
+ let baseUrl = url.substring(0, v1Index + 3);
+
+ // Check if the URL has '/openai' immediately after '/v1'.
+ const openaiIndex = url.indexOf('/openai', v1Index + 3);
+
+ // If '/openai' is found right after '/v1', include it in the base URL.
+ if (openaiIndex === v1Index + 3) {
+ // Find the next slash or the end of the URL after '/openai'.
+ const nextSlashIndex = url.indexOf('/', openaiIndex + 7);
+ if (nextSlashIndex === -1) {
+ // If there is no next slash, the rest of the URL is the base URL.
+ baseUrl = url.substring(0, openaiIndex + 7);
+ } else {
+ // If there is a next slash, the base URL goes up to but not including the slash.
+ baseUrl = url.substring(0, nextSlashIndex);
+ }
+ } else if (openaiIndex > 0) {
+ // If '/openai' is present but not immediately after '/v1', we need to include the reverse proxy pattern.
+ baseUrl = url.substring(0, openaiIndex + 7);
+ }
+
+ return baseUrl;
+}
+
+module.exports = extractBaseURL; // Export the function for use in your test file.
diff --git a/api/utils/extractBaseURL.spec.js b/api/utils/extractBaseURL.spec.js
new file mode 100644
index 0000000000..1403ac9968
--- /dev/null
+++ b/api/utils/extractBaseURL.spec.js
@@ -0,0 +1,56 @@
+const extractBaseURL = require('./extractBaseURL');
+
+describe('extractBaseURL', () => {
+ test('should extract base URL up to /v1 for standard endpoints', () => {
+ const url = 'https://localhost:8080/v1/chat/completions';
+ expect(extractBaseURL(url)).toBe('https://localhost:8080/v1');
+ });
+
+ test('should include /openai in the extracted URL when present', () => {
+ const url = 'https://localhost:8080/v1/openai';
+ expect(extractBaseURL(url)).toBe('https://localhost:8080/v1/openai');
+ });
+
+ test('should stop at /openai and not include any additional paths', () => {
+ const url = 'https://fake.open.ai/v1/openai/you-are-cool';
+ expect(extractBaseURL(url)).toBe('https://fake.open.ai/v1/openai');
+ });
+
+ test('should return the correct base URL for official openai endpoints', () => {
+ const url = 'https://api.openai.com/v1/chat/completions';
+ expect(extractBaseURL(url)).toBe('https://api.openai.com/v1');
+ });
+
+ test('should handle URLs with reverse proxy pattern correctly', () => {
+ const url = 'https://gateway.ai.cloudflare.com/v1/ACCOUNT_TAG/GATEWAY/openai/completions';
+ expect(extractBaseURL(url)).toBe(
+ 'https://gateway.ai.cloudflare.com/v1/ACCOUNT_TAG/GATEWAY/openai',
+ );
+ });
+
+ test('should return null if the URL does not match the expected pattern', () => {
+ const url = 'https://someotherdomain.com/notv1';
+ expect(extractBaseURL(url)).toBeNull();
+ });
+
+ // Test our JSDoc examples.
+ test('should extract base URL up to /v1 for open.ai standard endpoint', () => {
+ const url = 'https://open.ai/v1/chat';
+ expect(extractBaseURL(url)).toBe('https://open.ai/v1');
+ });
+
+ test('should extract base URL up to /v1 for open.ai standard endpoint with additional path', () => {
+ const url = 'https://open.ai/v1/chat/completions';
+ expect(extractBaseURL(url)).toBe('https://open.ai/v1');
+ });
+
+ test('should handle URLs with ACCOUNT/GATEWAY pattern followed by /openai', () => {
+ const url = 'https://open.ai/v1/ACCOUNT/GATEWAY/openai/completions';
+ expect(extractBaseURL(url)).toBe('https://open.ai/v1/ACCOUNT/GATEWAY/openai');
+ });
+
+ test('should include /openai in the extracted URL with additional segments', () => {
+ const url = 'https://open.ai/v1/hi/openai';
+ expect(extractBaseURL(url)).toBe('https://open.ai/v1/hi/openai');
+ });
+});
diff --git a/api/utils/index.js b/api/utils/index.js
index 3e1c4d0c44..f9194858e8 100644
--- a/api/utils/index.js
+++ b/api/utils/index.js
@@ -1,9 +1,11 @@
-const azureUtils = require('./azureUtils');
const tokenHelpers = require('./tokens');
+const azureUtils = require('./azureUtils');
+const extractBaseURL = require('./extractBaseURL');
const findMessageContent = require('./findMessageContent');
module.exports = {
...azureUtils,
...tokenHelpers,
+ extractBaseURL,
findMessageContent,
};