Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🛠️ refactor: Model Loading and Custom Endpoint Error Handling (#1849)

Danny Aviladd8038b37590
diff --git a/api/app/clients/OpenAIClient.js b/api/app/clients/OpenAIClient.js
index e913c6a994..4cd3df378f 100644
--- a/api/app/clients/OpenAIClient.js
+++ b/api/app/clients/OpenAIClient.js
@@ -994,6 +994,7 @@ ${convo}
}
let chatCompletion;
+ /** @type {OpenAI} */
const openai = new OpenAI({
apiKey: this.apiKey,
...opts,
@@ -1046,6 +1047,16 @@ ${convo}
.on('error', (err) => {
handleOpenAIErrors(err, errorCallback, 'stream');
})
+ .on('finalChatCompletion', (finalChatCompletion) => {
+ const finalMessage = finalChatCompletion?.choices?.[0]?.message;
+ if (finalMessage && finalMessage?.role !== 'assistant') {
+ finalChatCompletion.choices[0].message.role = 'assistant';
+ }
+
+ if (finalMessage && !finalMessage?.content?.trim()) {
+ finalChatCompletion.choices[0].message.content = intermediateReply;
+ }
+ })
.on('finalMessage', (message) => {
if (message?.role !== 'assistant') {
stream.messages.push({ role: 'assistant', content: intermediateReply });
@@ -1117,6 +1128,9 @@ ${convo}
err?.message?.includes(
'OpenAI error: Invalid final message: OpenAI expects final message to include role=assistant',
) ||
+ err?.message?.includes(
+ 'stream ended without producing a ChatCompletionMessage with role=assistant',
+ ) ||
err?.message?.includes('The server had an error processing your request') ||
err?.message?.includes('missing finish_reason') ||
err?.message?.includes('missing role') ||
diff --git a/api/server/controllers/ModelController.js b/api/server/controllers/ModelController.js
index 0025ef910d..e8da600d37 100644
--- a/api/server/controllers/ModelController.js
+++ b/api/server/controllers/ModelController.js
@@ -2,12 +2,16 @@ const { CacheKeys } = require('librechat-data-provider');
const { loadDefaultModels, loadConfigModels } = require('~/server/services/Config');
const { getLogStores } = require('~/cache');
-async function modelController(req, res) {
+/**
+ * Loads the models from the config.
+ * @param {Express.Request} req - The Express request object.
+ * @returns {Promise<TModelsConfig>} The models config.
+ */
+async function loadModels(req) {
const cache = getLogStores(CacheKeys.CONFIG_STORE);
const cachedModelsConfig = await cache.get(CacheKeys.MODELS_CONFIG);
if (cachedModelsConfig) {
- res.send(cachedModelsConfig);
- return;
+ return cachedModelsConfig;
}
const defaultModelsConfig = await loadDefaultModels(req);
const customModelsConfig = await loadConfigModels(req);
@@ -15,7 +19,12 @@ async function modelController(req, res) {
const modelConfig = { ...defaultModelsConfig, ...customModelsConfig };
await cache.set(CacheKeys.MODELS_CONFIG, modelConfig);
+ return modelConfig;
+}
+
+async function modelController(req, res) {
+ const modelConfig = await loadModels(req);
res.send(modelConfig);
}
-module.exports = modelController;
+module.exports = { modelController, loadModels };
diff --git a/api/server/middleware/validateModel.js b/api/server/middleware/validateModel.js
index 553097f99e..dc069b4ae3 100644
--- a/api/server/middleware/validateModel.js
+++ b/api/server/middleware/validateModel.js
@@ -1,4 +1,5 @@
const { EModelEndpoint, CacheKeys, ViolationTypes } = require('librechat-data-provider');
+const { loadModels } = require('~/server/controllers/ModelController');
const { logViolation, getLogStores } = require('~/cache');
const { handleError } = require('~/server/utils');
@@ -17,7 +18,11 @@ const validateModel = async (req, res, next) => {
}
const cache = getLogStores(CacheKeys.CONFIG_STORE);
- const modelsConfig = await cache.get(CacheKeys.MODELS_CONFIG);
+ let modelsConfig = await cache.get(CacheKeys.MODELS_CONFIG);
+ if (!modelsConfig) {
+ modelsConfig = await loadModels(req);
+ }
+
if (!modelsConfig) {
return handleError(res, { text: 'Models not loaded' });
}
diff --git a/api/server/routes/models.js b/api/server/routes/models.js
index 383a63c113..e3272087a7 100644
--- a/api/server/routes/models.js
+++ b/api/server/routes/models.js
@@ -1,8 +1,8 @@
const express = require('express');
-const router = express.Router();
-const controller = require('../controllers/ModelController');
-const { requireJwtAuth } = require('../middleware/');
+const { modelController } = require('~/server/controllers/ModelController');
+const { requireJwtAuth } = require('~/server/middleware/');
-router.get('/', requireJwtAuth, controller);
+const router = express.Router();
+router.get('/', requireJwtAuth, modelController);
module.exports = router;
diff --git a/api/server/services/Endpoints/custom/initializeClient.js b/api/server/services/Endpoints/custom/initializeClient.js
index c308bd6a71..da14b37dea 100644
--- a/api/server/services/Endpoints/custom/initializeClient.js
+++ b/api/server/services/Endpoints/custom/initializeClient.js
@@ -41,7 +41,7 @@ const initializeClient = async ({ req, res, endpointOption }) => {
const cache = getLogStores(CacheKeys.TOKEN_CONFIG);
let endpointTokenConfig = await cache.get(endpoint);
- if (!endpointTokenConfig) {
+ if (endpointConfig && endpointConfig.models.fetch && !endpointTokenConfig) {
await fetchModels({ apiKey: CUSTOM_API_KEY, baseURL: CUSTOM_BASE_URL, name: endpoint });
endpointTokenConfig = await cache.get(endpoint);
}
diff --git a/api/typedefs.js b/api/typedefs.js
index 6bc792cbe6..b8eef88a50 100644
--- a/api/typedefs.js
+++ b/api/typedefs.js
@@ -32,6 +32,12 @@
* @memberof typedefs
*/
+/**
+ * @exports TModelsConfig
+ * @typedef {import('librechat-data-provider').TModelsConfig} TModelsConfig
+ * @memberof typedefs
+ */
+
/**
* @exports TPlugin
* @typedef {import('librechat-data-provider').TPlugin} TPlugin
diff --git a/docs/features/mod_system.md b/docs/features/mod_system.md
index 3bef6b11dd..6c9dc88490 100644
--- a/docs/features/mod_system.md
+++ b/docs/features/mod_system.md
@@ -31,11 +31,14 @@ The project's current rate limiters are as follows (see below under setup for de
- Login and registration rate limiting
- [optional] Concurrent Message limiting (only X messages at a time per user)
- [optional] Message limiting (how often a user can send a message, configurable by IP and User)
+- [optional] File Upload limiting: configurable through [`librechat.yaml` config file](https://docs.librechat.ai/install/configuration/custom_config.html#rate-limiting).
### Setup
The following are all of the related env variables to make use of and configure the mod system. Note this is also found in the [/.env.example](https://github.com/danny-avila/LibreChat/blob/main/.env.example) file, to be set in your own `.env` file.
+**Note:** currently, most of these values are configured through the .env file, but they may soon migrate to be exclusively configured from the [`librechat.yaml` config file](https://docs.librechat.ai/install/configuration/custom_config.html#rate-limiting).
+
```bash
BAN_VIOLATIONS=true # Whether or not to enable banning users for violations (they will still be logged)
BAN_DURATION=1000 * 60 * 60 * 2 # how long the user and associated IP are banned for