Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🎭 test: Run Mock E2E Suite Through `createRun` With In-Process Fake Model (#13508)

Danny Avilaa1bfa3b298b8
diff --git a/.github/workflows/playwright-mock.yml b/.github/workflows/playwright-mock.yml
index 91a2f9910b..f0d0d1323d 100644
--- a/.github/workflows/playwright-mock.yml
+++ b/.github/workflows/playwright-mock.yml
@@ -1,4 +1,4 @@
-name: Playwright E2E (Mock LLM)
+name: Playwright E2E Tests
on:
pull_request:
@@ -22,7 +22,6 @@ env:
jobs:
e2e:
- name: Tier-1 smoke (headless Chrome)
runs-on: ubuntu-latest
timeout-minutes: 30
env:
diff --git a/api/package.json b/api/package.json
index 9c4da2df2e..652cec9ccd 100644
--- a/api/package.json
+++ b/api/package.json
@@ -46,7 +46,7 @@
"@azure/storage-blob": "^12.30.0",
"@google/genai": "^2.0.1",
"@keyv/redis": "^4.3.3",
- "@librechat/agents": "^3.2.1",
+ "@librechat/agents": "^3.2.2",
"@librechat/api": "*",
"@librechat/data-schemas": "*",
"@microsoft/microsoft-graph-client": "^3.0.7",
diff --git a/e2e/README.md b/e2e/README.md
index 822297103f..86ec912447 100644
--- a/e2e/README.md
+++ b/e2e/README.md
@@ -1,6 +1,6 @@
# LibreChat e2e
-The mock e2e profile is the safest default for generated tests. It starts LibreChat with `e2e/config/librechat.e2e.yaml`, points custom endpoints at the local mock LLM server, creates an authenticated e2e user, and avoids real provider credentials.
+The mock e2e profile is the safest default for generated tests. It starts LibreChat with `e2e/config/librechat.e2e.yaml`, injects an in-process fake LLM (via `LIBRECHAT_TEST_RUN_HOOK`), creates an authenticated e2e user, and avoids real provider credentials.
## Recording Tests
@@ -10,9 +10,9 @@ Use Playwright codegen when you want to turn an exploratory browser session into
npm run e2e:record
```
-That command builds the app, starts the mock LLM and LibreChat test server when needed, writes `e2e/storageState.json`, and opens Playwright codegen at `/c/new`. The npm script uses `http://localhost:3333` so it does not collide with a normal dev server on `3080`. Raw recordings are written to `e2e/recordings/` and ignored by git.
+That command builds the app, starts the LibreChat test server (with an in-process fake LLM) when needed, writes `e2e/storageState.json`, and opens Playwright codegen at `/c/new`. The npm script uses `http://localhost:3333` so it does not collide with a normal dev server on `3080`. Raw recordings are written to `e2e/recordings/` and ignored by git.
-For a real local LibreChat config instead of the mock LLM profile:
+For a real local LibreChat config instead of the mock profile:
```sh
npm run e2e:record:local
diff --git a/e2e/config/librechat.e2e.yaml b/e2e/config/librechat.e2e.yaml
index e7e9e62271..8f73e53e8c 100644
--- a/e2e/config/librechat.e2e.yaml
+++ b/e2e/config/librechat.e2e.yaml
@@ -1,5 +1,6 @@
-# Credential-free e2e config template. e2e/playwright.config.mock.ts writes
-# an ignored runtime copy and rewrites the mock LLM port from MOCK_LLM_PORT.
+# Credential-free e2e config template. e2e/playwright.config.mock.ts copies this
+# to an ignored runtime path; the in-process fake model (e2e/setup/fake-model.js)
+# overrides each run, so the placeholder baseURL below is never contacted.
version: 1.3.11
cache: true
diff --git a/e2e/playwright.config.mock.ts b/e2e/playwright.config.mock.ts
index a78d51817c..08284efeed 100644
--- a/e2e/playwright.config.mock.ts
+++ b/e2e/playwright.config.mock.ts
@@ -5,15 +5,12 @@ import { getLocalE2EEnv, getE2EBaseURL } from './setup/env';
const rootPath = path.resolve(__dirname, '..');
const serverPath = path.resolve(rootPath, 'e2e/setup/start-server.js');
-const mockLlmPath = path.resolve(rootPath, 'e2e/setup/mock-llm-server.js');
+const fakeModelHookPath = path.resolve(rootPath, 'e2e/setup/fake-model.js');
const configTemplatePath = path.resolve(rootPath, 'e2e/config/librechat.e2e.yaml');
const configPath = path.resolve(rootPath, 'e2e/.generated/librechat.e2e.yaml');
const reportPath = path.resolve(rootPath, 'e2e/playwright-report');
const baseURL = getE2EBaseURL();
-const mockLlmPort = getMockLlmPort();
-const defaultMockLlmBaseURL = 'http://127.0.0.1:8889/v1';
-const mockLlmBaseURL = `http://127.0.0.1:${mockLlmPort}/v1`;
const chromiumChannel = process.env.E2E_CHROMIUM_CHANNEL || undefined;
const vanillaOverrides = {
@@ -30,7 +27,8 @@ const vanillaOverrides = {
const baseEnv = {
...getLocalE2EEnv(),
CONFIG_PATH: configPath,
- MOCK_LLM_PORT: mockLlmPort,
+ /** Loaded in-process by `@librechat/api`'s `createRun` to swap in a fake model. */
+ LIBRECHAT_TEST_RUN_HOOK: fakeModelHookPath,
...vanillaOverrides,
};
@@ -41,23 +39,15 @@ const preservedCredentialEnvKeys = new Set([
'E2E_USER_B_PASSWORD',
]);
-function getMockLlmPort() {
- const port = process.env.MOCK_LLM_PORT ?? '8889';
- if (!/^\d+$/.test(port)) {
- throw new Error('MOCK_LLM_PORT must be a numeric port');
- }
- return port;
-}
-
+/**
+ * The custom endpoints in the template point at an unreachable baseURL; the fake
+ * model injected via `LIBRECHAT_TEST_RUN_HOOK` overrides the run before any
+ * request is made, so no real (or mock HTTP) provider is contacted.
+ */
function writeRuntimeMockConfig() {
const template = fs.readFileSync(configTemplatePath, 'utf8');
-
- if (!template.includes(defaultMockLlmBaseURL)) {
- throw new Error(`Expected mock config template to include ${defaultMockLlmBaseURL}`);
- }
-
fs.mkdirSync(path.dirname(configPath), { recursive: true });
- fs.writeFileSync(configPath, template.replaceAll(defaultMockLlmBaseURL, mockLlmBaseURL));
+ fs.writeFileSync(configPath, template);
}
function neutralizeCredentialEnv(env: NodeJS.ProcessEnv, keep: Set<string>) {
@@ -128,14 +118,6 @@ export default defineConfig({
},
],
webServer: [
- {
- command: `node ${mockLlmPath}`,
- cwd: rootPath,
- url: `http://127.0.0.1:${mockLlmPort}/health`,
- stdout: 'pipe',
- timeout: 30_000,
- reuseExistingServer: false,
- },
{
command: `node ${serverPath}`,
cwd: rootPath,
diff --git a/e2e/setup/fake-model.js b/e2e/setup/fake-model.js
new file mode 100644
index 0000000000..04a2f9b47b
--- /dev/null
+++ b/e2e/setup/fake-model.js
@@ -0,0 +1,213 @@
+/**
+ * In-process fake LLM for credential-free e2e tests. Loaded by `@librechat/api`'s
+ * `createRun` via the `LIBRECHAT_TEST_RUN_HOOK` env var (set by the mock
+ * Playwright config and the `--profile=mock` recorder), it swaps the run's model
+ * for the agents package's own `FakeChatModel` through
+ * `run.Graph.overrideTestModel(...)`.
+ *
+ * This exercises the real `Run.create` -> graph -> tool-node pipeline end to end
+ * without a live provider or a standalone HTTP mock server: responses are decided
+ * from the conversation and the agents' advertised tools.
+ */
+const MOCK_REPLY = process.env.MOCK_LLM_REPLY || 'E2E mock reply: pong';
+const CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_CHUNK_DELAY_MS) || 10;
+
+const CREATE_SKILL_MARKER = 'E2E_CREATE_SKILL:';
+const EDIT_SKILL_MARKER = 'E2E_EDIT_SKILL:';
+const CREATE_FILE_AUTHORING_FINAL_TEXT = 'E2E file authoring complete';
+const EDIT_FILE_AUTHORING_FINAL_TEXT = 'E2E file edit complete';
+const CREATE_FILE_TOOL_NAME = 'create_file';
+const EDIT_FILE_TOOL_NAME = 'edit_file';
+const BASH_TOOL_NAME = 'bash_tool';
+const CREATE_SKILL_TOOL_CALL_ID = 'call_e2e_create_skill';
+const EDIT_SKILL_TOOL_CALL_ID = 'call_e2e_edit_skill';
+const SKILL_DESCRIPTION =
+ 'Use this skill to verify LibreChat skill file authoring in mock end-to-end tests.';
+const EDITED_SKILL_DESCRIPTION =
+ 'Use this edited skill to verify LibreChat skill file authoring in mock end-to-end tests.';
+
+function messageType(message) {
+ if (typeof message.getType === 'function') {
+ return message.getType();
+ }
+ if (typeof message._getType === 'function') {
+ return message._getType();
+ }
+ return message.role || message.type || '';
+}
+
+function getContentText(content) {
+ if (typeof content === 'string') {
+ return content;
+ }
+ if (!Array.isArray(content)) {
+ return '';
+ }
+ return content
+ .map((part) => {
+ if (typeof part === 'string') {
+ return part;
+ }
+ if (part && typeof part === 'object' && typeof part.text === 'string') {
+ return part.text;
+ }
+ return '';
+ })
+ .join('\n');
+}
+
+function getLatestUserText(messages) {
+ if (!Array.isArray(messages)) {
+ return '';
+ }
+ for (let index = messages.length - 1; index >= 0; index--) {
+ const message = messages[index];
+ if (!message) {
+ continue;
+ }
+ const type = messageType(message);
+ if (type === 'human' || type === 'user') {
+ return getContentText(message.content);
+ }
+ }
+ return '';
+}
+
+function getRequestedSkillName(text, marker) {
+ const markerIndex = text.indexOf(marker);
+ if (markerIndex === -1) {
+ return '';
+ }
+ const afterMarker = text.slice(markerIndex + marker.length);
+ return afterMarker.match(/[a-z0-9][a-z0-9-]*/)?.[0] ?? '';
+}
+
+function collectToolNames(agents) {
+ const names = new Set();
+ const add = (name) => {
+ if (typeof name === 'string' && name) {
+ names.add(name);
+ }
+ };
+ for (const agent of agents ?? []) {
+ if (!agent) {
+ continue;
+ }
+ for (const tool of agent.tools ?? []) {
+ add(tool?.name);
+ }
+ for (const def of agent.toolDefinitions ?? []) {
+ add(def?.name);
+ }
+ if (agent.toolRegistry && typeof agent.toolRegistry.keys === 'function') {
+ for (const name of agent.toolRegistry.keys()) {
+ add(name);
+ }
+ }
+ }
+ return names;
+}
+
+function buildSkillBody(skillName) {
+ return `---
+name: ${skillName}
+description: ${SKILL_DESCRIPTION}
+---
+
+# ${skillName}
+
+Created by the Playwright mock e2e suite to verify host file authoring without code execution.`;
+}
+
+function buildCreateSkillArgs(skillName) {
+ return {
+ file_path: `skills/${skillName}/SKILL.md`,
+ content: buildSkillBody(skillName),
+ overwrite: false,
+ };
+}
+
+function buildEditSkillArgs(skillName) {
+ return {
+ file_path: `skills/${skillName}/SKILL.md`,
+ old_text: `description: ${SKILL_DESCRIPTION}`,
+ new_text: `description: ${EDITED_SKILL_DESCRIPTION}`,
+ };
+}
+
+/**
+ * Pick the fake-model script for a skill file-authoring turn. The graph runs two
+ * model turns: turn 1 streams the (empty) preamble and emits the tool call, the
+ * tool node writes the SKILL.md, then turn 2 streams the final text. The guards
+ * assert the feature advertised the host file-authoring tool and did NOT enable
+ * code execution.
+ */
+function fileAuthoringResponses(operation, toolNames) {
+ if (!toolNames.has(operation.toolName)) {
+ return {
+ responses: [`E2E file authoring unavailable: ${operation.toolName} was not advertised.`],
+ };
+ }
+ if (toolNames.has(BASH_TOOL_NAME)) {
+ return {
+ responses: [`E2E file authoring unavailable: ${BASH_TOOL_NAME} was unexpectedly advertised.`],
+ };
+ }
+ return {
+ responses: ['', `${operation.finalText}: ${operation.skillName}`],
+ toolCalls: [
+ {
+ id: operation.toolCallId,
+ name: operation.toolName,
+ args: operation.args,
+ type: 'tool_call',
+ },
+ ],
+ };
+}
+
+function resolveResponses(text, toolNames) {
+ const createSkillName = getRequestedSkillName(text, CREATE_SKILL_MARKER);
+ if (createSkillName) {
+ return fileAuthoringResponses(
+ {
+ skillName: createSkillName,
+ toolName: CREATE_FILE_TOOL_NAME,
+ toolCallId: CREATE_SKILL_TOOL_CALL_ID,
+ finalText: CREATE_FILE_AUTHORING_FINAL_TEXT,
+ args: buildCreateSkillArgs(createSkillName),
+ },
+ toolNames,
+ );
+ }
+
+ const editSkillName = getRequestedSkillName(text, EDIT_SKILL_MARKER);
+ if (editSkillName) {
+ return fileAuthoringResponses(
+ {
+ skillName: editSkillName,
+ toolName: EDIT_FILE_TOOL_NAME,
+ toolCallId: EDIT_SKILL_TOOL_CALL_ID,
+ finalText: EDIT_FILE_AUTHORING_FINAL_TEXT,
+ args: buildEditSkillArgs(editSkillName),
+ },
+ toolNames,
+ );
+ }
+
+ return { responses: [MOCK_REPLY] };
+}
+
+/** @type {import('@librechat/api').TestRunHook} */
+module.exports = function fakeModelHook(run, context) {
+ const graph = run?.Graph;
+ if (!graph || typeof graph.overrideTestModel !== 'function') {
+ console.warn('[e2e] fake-model hook: run.Graph.overrideTestModel unavailable');
+ return;
+ }
+
+ const text = getLatestUserText(context?.messages);
+ const toolNames = collectToolNames(context?.agents);
+ const { responses, toolCalls } = resolveResponses(text, toolNames);
+ graph.overrideTestModel(responses, CHUNK_DELAY_MS, toolCalls);
+};
diff --git a/e2e/setup/mock-llm-server.js b/e2e/setup/mock-llm-server.js
deleted file mode 100644
index 4f34190e9c..0000000000
--- a/e2e/setup/mock-llm-server.js
+++ /dev/null
@@ -1,392 +0,0 @@
-/**
- * OpenAI-compatible mock server for credential-free e2e tests. Answers
- * `${baseURL}/chat/completions` with deterministic content. Run standalone
- * (Playwright `webServer`) or import `startMockLlm()` for programmatic control.
- */
-const http = require('http');
-
-const DEFAULT_PORT = 8889;
-const MOCK_REPLY = process.env.MOCK_LLM_REPLY || 'E2E mock reply: pong';
-const MODEL_FALLBACK = 'mock-model';
-const STREAM_CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_CHUNK_DELAY_MS) || 60;
-const CREATE_SKILL_MARKER = 'E2E_CREATE_SKILL:';
-const EDIT_SKILL_MARKER = 'E2E_EDIT_SKILL:';
-const CREATE_FILE_AUTHORING_FINAL_TEXT = 'E2E file authoring complete';
-const EDIT_FILE_AUTHORING_FINAL_TEXT = 'E2E file edit complete';
-const CREATE_FILE_TOOL_NAME = 'create_file';
-const EDIT_FILE_TOOL_NAME = 'edit_file';
-const BASH_TOOL_NAME = 'bash_tool';
-const CREATE_SKILL_TOOL_CALL_ID = 'call_e2e_create_skill';
-const EDIT_SKILL_TOOL_CALL_ID = 'call_e2e_edit_skill';
-const SKILL_DESCRIPTION =
- 'Use this skill to verify LibreChat skill file authoring in mock end-to-end tests.';
-const EDITED_SKILL_DESCRIPTION =
- 'Use this edited skill to verify LibreChat skill file authoring in mock end-to-end tests.';
-
-const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
-
-function readJsonBody(req) {
- return new Promise((resolve) => {
- let raw = '';
- req.on('data', (chunk) => {
- raw += chunk;
- });
- req.on('end', () => {
- try {
- resolve(raw ? JSON.parse(raw) : {});
- } catch {
- resolve({});
- }
- });
- });
-}
-
-function toChunks(text) {
- return text.match(/\S+\s*/g) || [text];
-}
-
-function getToolName(tool) {
- if (!tool || typeof tool !== 'object') {
- return '';
- }
- if (typeof tool.name === 'string') {
- return tool.name;
- }
- if (tool.function && typeof tool.function.name === 'string') {
- return tool.function.name;
- }
- return '';
-}
-
-function hasTool(body, name) {
- const tools = Array.isArray(body.tools) ? body.tools : [];
- const functions = Array.isArray(body.functions) ? body.functions : [];
- return [...tools, ...functions].some((tool) => getToolName(tool) === name);
-}
-
-function getContentText(content) {
- if (typeof content === 'string') {
- return content;
- }
- if (!Array.isArray(content)) {
- return '';
- }
- return content
- .map((part) => {
- if (typeof part === 'string') {
- return part;
- }
- if (part && typeof part === 'object' && typeof part.text === 'string') {
- return part.text;
- }
- return '';
- })
- .join('\n');
-}
-
-function getLatestUserText(body) {
- const messages = Array.isArray(body.messages) ? body.messages : [];
- for (let index = messages.length - 1; index >= 0; index--) {
- const message = messages[index];
- if (!message || message.role !== 'user') {
- continue;
- }
- return getContentText(message.content);
- }
- return '';
-}
-
-function getRequestedSkillName(body, marker) {
- const text = getLatestUserText(body);
- const markerIndex = text.indexOf(marker);
- if (markerIndex === -1) {
- return '';
- }
- const afterMarker = text.slice(markerIndex + marker.length);
- const skillName = afterMarker.match(/[a-z0-9][a-z0-9-]*/)?.[0];
- if (skillName) {
- return skillName;
- }
- return '';
-}
-
-function hasFileAuthoringToolResult(body, toolCallId) {
- const messages = Array.isArray(body.messages) ? body.messages : [];
- return messages.some(
- (message) => message && message.role === 'tool' && message.tool_call_id === toolCallId,
- );
-}
-
-function buildSkillBody(skillName) {
- return `---
-name: ${skillName}
-description: ${SKILL_DESCRIPTION}
----
-
-# ${skillName}
-
-Created by the Playwright mock e2e suite to verify host file authoring without code execution.`;
-}
-
-function buildCreateSkillArgs(skillName) {
- return {
- file_path: `skills/${skillName}/SKILL.md`,
- content: buildSkillBody(skillName),
- overwrite: false,
- };
-}
-
-function buildEditSkillArgs(skillName) {
- return {
- file_path: `skills/${skillName}/SKILL.md`,
- old_text: `description: ${SKILL_DESCRIPTION}`,
- new_text: `description: ${EDITED_SKILL_DESCRIPTION}`,
- };
-}
-
-async function streamTextCompletion(res, model, text = MOCK_REPLY) {
- res.writeHead(200, {
- 'Content-Type': 'text/event-stream',
- 'Cache-Control': 'no-cache',
- Connection: 'keep-alive',
- });
- if (typeof res.flushHeaders === 'function') {
- res.flushHeaders();
- }
-
- const id = 'chatcmpl-e2e-mock';
- const created = 1700000000;
- const base = { id, object: 'chat.completion.chunk', created, model };
-
- const send = (delta, finishReason = null) => {
- const payload = {
- ...base,
- choices: [{ index: 0, delta, finish_reason: finishReason }],
- };
- res.write(`data: ${JSON.stringify(payload)}\n\n`);
- };
-
- send({ role: 'assistant', content: '' });
- for (const chunk of toChunks(text)) {
- await delay(STREAM_CHUNK_DELAY_MS);
- send({ content: chunk });
- }
- send({}, 'stop');
- res.write('data: [DONE]\n\n');
- res.end();
-}
-
-async function streamToolCall(res, model, { toolName, toolCallId, args }) {
- res.writeHead(200, {
- 'Content-Type': 'text/event-stream',
- 'Cache-Control': 'no-cache',
- Connection: 'keep-alive',
- });
- if (typeof res.flushHeaders === 'function') {
- res.flushHeaders();
- }
-
- const id = 'chatcmpl-e2e-mock';
- const created = 1700000000;
- const base = { id, object: 'chat.completion.chunk', created, model };
- const send = (delta, finishReason = null) => {
- const payload = {
- ...base,
- choices: [{ index: 0, delta, finish_reason: finishReason }],
- };
- res.write(`data: ${JSON.stringify(payload)}\n\n`);
- };
-
- send({ role: 'assistant', content: '' });
- await delay(STREAM_CHUNK_DELAY_MS);
- send({
- tool_calls: [
- {
- index: 0,
- id: toolCallId,
- type: 'function',
- function: { name: toolName, arguments: '' },
- },
- ],
- });
- await delay(STREAM_CHUNK_DELAY_MS);
- send({
- tool_calls: [
- {
- index: 0,
- function: { arguments: JSON.stringify(args) },
- },
- ],
- });
- send({}, 'tool_calls');
- res.write('data: [DONE]\n\n');
- res.end();
-}
-
-function jsonCompletion(res, model, content = MOCK_REPLY) {
- const payload = {
- id: 'chatcmpl-e2e-mock',
- object: 'chat.completion',
- created: 1700000000,
- model,
- choices: [
- {
- index: 0,
- message: { role: 'assistant', content },
- finish_reason: 'stop',
- },
- ],
- usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
- };
- res.writeHead(200, { 'Content-Type': 'application/json' });
- res.end(JSON.stringify(payload));
-}
-
-function jsonToolCall(res, model, { toolName, toolCallId, args }) {
- const payload = {
- id: 'chatcmpl-e2e-mock',
- object: 'chat.completion',
- created: 1700000000,
- model,
- choices: [
- {
- index: 0,
- message: {
- role: 'assistant',
- content: null,
- tool_calls: [
- {
- id: toolCallId,
- type: 'function',
- function: {
- name: toolName,
- arguments: JSON.stringify(args),
- },
- },
- ],
- },
- finish_reason: 'tool_calls',
- },
- ],
- usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
- };
- res.writeHead(200, { 'Content-Type': 'application/json' });
- res.end(JSON.stringify(payload));
-}
-
-async function handleFileAuthoringOperation(body, res, model, operation) {
- const finalText = `${operation.finalText}: ${operation.skillName}`;
- if (hasFileAuthoringToolResult(body, operation.toolCallId)) {
- if (body.stream) {
- await streamTextCompletion(res, model, finalText);
- } else {
- jsonCompletion(res, model, finalText);
- }
- return true;
- }
-
- if (!hasTool(body, operation.toolName)) {
- const unavailable = `E2E file authoring unavailable: ${operation.toolName} was not advertised.`;
- if (body.stream) {
- await streamTextCompletion(res, model, unavailable);
- } else {
- jsonCompletion(res, model, unavailable);
- }
- return true;
- }
-
- if (hasTool(body, BASH_TOOL_NAME)) {
- const unexpected = `E2E file authoring unavailable: ${BASH_TOOL_NAME} was unexpectedly advertised.`;
- if (body.stream) {
- await streamTextCompletion(res, model, unexpected);
- } else {
- jsonCompletion(res, model, unexpected);
- }
- return true;
- }
-
- const toolCall = {
- toolName: operation.toolName,
- toolCallId: operation.toolCallId,
- args: operation.args,
- };
- if (body.stream) {
- await streamToolCall(res, model, toolCall);
- } else {
- jsonToolCall(res, model, toolCall);
- }
- return true;
-}
-
-async function handleFileAuthoringE2E(body, res, model) {
- const createSkillName = getRequestedSkillName(body, CREATE_SKILL_MARKER);
- if (createSkillName) {
- return await handleFileAuthoringOperation(body, res, model, {
- skillName: createSkillName,
- toolName: CREATE_FILE_TOOL_NAME,
- toolCallId: CREATE_SKILL_TOOL_CALL_ID,
- finalText: CREATE_FILE_AUTHORING_FINAL_TEXT,
- args: buildCreateSkillArgs(createSkillName),
- });
- }
-
- const editSkillName = getRequestedSkillName(body, EDIT_SKILL_MARKER);
- if (editSkillName) {
- return await handleFileAuthoringOperation(body, res, model, {
- skillName: editSkillName,
- toolName: EDIT_FILE_TOOL_NAME,
- toolCallId: EDIT_SKILL_TOOL_CALL_ID,
- finalText: EDIT_FILE_AUTHORING_FINAL_TEXT,
- args: buildEditSkillArgs(editSkillName),
- });
- }
-
- return false;
-}
-
-async function handleRequest(req, res) {
- if (req.method === 'GET' && (req.url === '/health' || req.url === '/')) {
- res.writeHead(200, { 'Content-Type': 'application/json' });
- res.end(JSON.stringify({ status: 'ok' }));
- return;
- }
-
- if (req.method === 'POST' && req.url && req.url.endsWith('/chat/completions')) {
- const body = await readJsonBody(req);
- const model = body.model || MODEL_FALLBACK;
- if (await handleFileAuthoringE2E(body, res, model)) {
- return;
- }
- if (body.stream) {
- await streamTextCompletion(res, model);
- } else {
- jsonCompletion(res, model);
- }
- return;
- }
-
- res.writeHead(404, { 'Content-Type': 'application/json' });
- res.end(JSON.stringify({ error: 'not found' }));
-}
-
-function startMockLlm(port = Number(process.env.MOCK_LLM_PORT) || DEFAULT_PORT) {
- const server = http.createServer((req, res) => {
- handleRequest(req, res).catch(() => {
- res.writeHead(500, { 'Content-Type': 'application/json' });
- res.end(JSON.stringify({ error: 'mock server error' }));
- });
- });
-
- return new Promise((resolve) => {
- server.listen(port, '127.0.0.1', () => {
- console.log(`[e2e] Mock LLM server listening on http://127.0.0.1:${port}`);
- resolve(server);
- });
- });
-}
-
-if (require.main === module) {
- startMockLlm();
-}
-
-module.exports = { startMockLlm, MOCK_REPLY };
diff --git a/e2e/setup/record.js b/e2e/setup/record.js
index 6bbb1d0c98..70052a2b78 100644
--- a/e2e/setup/record.js
+++ b/e2e/setup/record.js
@@ -8,9 +8,7 @@ const baseURL = process.env.E2E_BASE_URL || 'http://localhost:3080';
const storageStatePath = path.resolve(rootPath, 'e2e/storageState.json');
const configTemplatePath = path.resolve(rootPath, 'e2e/config/librechat.e2e.yaml');
const configPath = path.resolve(rootPath, 'e2e/.generated/librechat.e2e.yaml');
-const defaultMockLlmBaseURL = 'http://127.0.0.1:8889/v1';
-const mockLlmPort = getMockLlmPort();
-const mockLlmBaseURL = `http://127.0.0.1:${mockLlmPort}/v1`;
+const fakeModelHookPath = path.resolve(rootPath, 'e2e/setup/fake-model.js');
const defaultUser = {
name: 'Test User',
@@ -49,7 +47,8 @@ const rateLimitOverrides = {
const mockOverrides = {
CONFIG_PATH: configPath,
- MOCK_LLM_PORT: mockLlmPort,
+ /** Loaded in-process by `@librechat/api`'s `createRun` to swap in a fake model. */
+ LIBRECHAT_TEST_RUN_HOOK: fakeModelHookPath,
OPENAI_API_KEY: 'user_provided',
TENANT_ISOLATION_STRICT: 'false',
OPENID_CLIENT_ID: '',
@@ -75,14 +74,6 @@ const preservedCredentialEnvKeys = new Set([
]);
const npxBin = process.platform === 'win32' ? 'npx.cmd' : 'npx';
-function getMockLlmPort() {
- const port = process.env.MOCK_LLM_PORT || '8889';
- if (!/^\d+$/.test(port)) {
- throw new Error('MOCK_LLM_PORT must be a numeric port');
- }
- return port;
-}
-
function appURL(pathname = '') {
const normalizedBaseURL = baseURL.endsWith('/') ? baseURL : `${baseURL}/`;
return new URL(pathname.replace(/^\/+/, ''), normalizedBaseURL).toString();
@@ -181,13 +172,8 @@ function formatDate(date) {
function writeRuntimeMockConfig() {
const template = fs.readFileSync(configTemplatePath, 'utf8');
-
- if (!template.includes(defaultMockLlmBaseURL)) {
- throw new Error(`Expected mock config template to include ${defaultMockLlmBaseURL}`);
- }
-
fs.mkdirSync(path.dirname(configPath), { recursive: true });
- fs.writeFileSync(configPath, template.replaceAll(defaultMockLlmBaseURL, mockLlmBaseURL));
+ fs.writeFileSync(configPath, template);
}
function parseArgs(argv) {
@@ -406,21 +392,6 @@ async function main() {
try {
if (options.profile === 'mock') {
writeRuntimeMockConfig();
-
- const mockURL = `http://127.0.0.1:${mockLlmPort}/health`;
- if (!(await waitForURL(mockURL, 1000))) {
- children.push(
- spawnProcess(
- 'mock-llm',
- 'node',
- [path.resolve(rootPath, 'e2e/setup/mock-llm-server.js')],
- env,
- ),
- );
- if (!(await waitForURL(mockURL, 30000))) {
- throw new Error(`Mock LLM server did not become ready at ${mockURL}`);
- }
- }
}
if (await waitForURL(baseURL, 1000)) {
diff --git a/package-lock.json b/package-lock.json
index e5b549fa47..12e9841163 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -61,7 +61,7 @@
"@azure/storage-blob": "^12.30.0",
"@google/genai": "^2.0.1",
"@keyv/redis": "^4.3.3",
- "@librechat/agents": "^3.2.1",
+ "@librechat/agents": "^3.2.2",
"@librechat/api": "*",
"@librechat/data-schemas": "*",
"@microsoft/microsoft-graph-client": "^3.0.7",
@@ -11734,9 +11734,9 @@
}
},
"node_modules/@librechat/agents": {
- "version": "3.2.1",
- "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-3.2.1.tgz",
- "integrity": "sha512-CusD5Iif0rd/mTAC/3J32ZZwm0EjArRkOUGEXQoMWlh5hrU2pGw/XjNK54GwSaqOAKV31bD569BhJ12BR2WXBQ==",
+ "version": "3.2.2",
+ "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-3.2.2.tgz",
+ "integrity": "sha512-ibjAoUCWQ4zgIJfnhAr3O8JkUfRXbcvyH2/4NFNc9YgRHvxS5tPoJAWUnBmC6t0AOt89vaJAQQwTMWo2YEB9qw==",
"license": "MIT",
"dependencies": {
"@anthropic-ai/sdk": "^0.92.0",
@@ -44016,7 +44016,7 @@
"@azure/storage-blob": "^12.30.0",
"@google/genai": "^2.0.1",
"@keyv/redis": "^4.3.3",
- "@librechat/agents": "^3.2.1",
+ "@librechat/agents": "^3.2.2",
"@librechat/data-schemas": "*",
"@modelcontextprotocol/sdk": "^1.29.0",
"@opentelemetry/api": "^1.9.0",
diff --git a/packages/api/package.json b/packages/api/package.json
index e8fe6d74ad..d1cc530352 100644
--- a/packages/api/package.json
+++ b/packages/api/package.json
@@ -104,7 +104,7 @@
"@azure/storage-blob": "^12.30.0",
"@google/genai": "^2.0.1",
"@keyv/redis": "^4.3.3",
- "@librechat/agents": "^3.2.1",
+ "@librechat/agents": "^3.2.2",
"@librechat/data-schemas": "*",
"@modelcontextprotocol/sdk": "^1.29.0",
"@opentelemetry/api": "^1.9.0",
diff --git a/packages/api/src/agents/index.ts b/packages/api/src/agents/index.ts
index 31d2485a01..14e0f480fd 100644
--- a/packages/api/src/agents/index.ts
+++ b/packages/api/src/agents/index.ts
@@ -22,6 +22,7 @@ export * from './skillConfigurable';
export * from './skillFiles';
export * from './codeFilesSession';
export * from './run';
+export * from './testHook';
export * from './tools';
export * from './validation';
export * from './added';
diff --git a/packages/api/src/agents/run.ts b/packages/api/src/agents/run.ts
index 85b614f4f7..2549ec7984 100644
--- a/packages/api/src/agents/run.ts
+++ b/packages/api/src/agents/run.ts
@@ -35,6 +35,7 @@ import type * as t from '~/types';
import { getProviderConfig } from '~/endpoints/config/providers';
import { resolveHeaders, createSafeUser } from '~/utils/env';
import { getOpenAIConfig } from '~/endpoints/openai/config';
+import { applyTestRunHook } from '~/agents/testHook';
import { isUserProvided } from '~/utils/common';
/** Expected shape of JSON tool search results */
@@ -1030,7 +1031,7 @@ export async function createRun({
*/
const enableToolOutputReferences = anyAgentHasCodeEnv(agents);
- return Run.create({
+ const run = await Run.create({
runId,
graphConfig,
tokenCounter,
@@ -1043,4 +1044,7 @@ export async function createRun({
toolOutputReferences: { enabled: true },
}),
});
+
+ applyTestRunHook(run, { messages, agents });
+ return run;
}
diff --git a/packages/api/src/agents/testHook.ts b/packages/api/src/agents/testHook.ts
new file mode 100644
index 0000000000..1599669c82
--- /dev/null
+++ b/packages/api/src/agents/testHook.ts
@@ -0,0 +1,40 @@
+import { logger } from '@librechat/data-schemas';
+import type { Run, IState } from '@librechat/agents';
+import type { BaseMessage } from '@librechat/agents/langchain/messages';
+
+/**
+ * Context handed to a test run hook so it can shape fake-model behavior from
+ * the conversation and the agents' advertised tools.
+ */
+export interface TestRunHookContext {
+ messages?: BaseMessage[];
+ agents: ReadonlyArray<{ tools?: ReadonlyArray<{ name: string }> }>;
+}
+
+export type TestRunHook = (run: Run<IState>, context: TestRunHookContext) => void;
+
+/**
+ * Env-gated extension point used only by the e2e harness. When
+ * `LIBRECHAT_TEST_RUN_HOOK` points at a module, it is loaded and invoked with
+ * the freshly created run so a test can swap in a fake model via
+ * `run.Graph.overrideTestModel(...)` instead of reaching a live provider. A
+ * no-op (returns immediately) in normal operation since the env var is unset.
+ */
+export function applyTestRunHook(run: Run<IState>, context: TestRunHookContext): void {
+ const hookPath = process.env.LIBRECHAT_TEST_RUN_HOOK;
+ if (!hookPath) {
+ return;
+ }
+ try {
+ // eslint-disable-next-line @typescript-eslint/no-require-imports
+ const loaded = require(hookPath) as TestRunHook | { default?: TestRunHook };
+ const hook = typeof loaded === 'function' ? loaded : loaded.default;
+ if (typeof hook !== 'function') {
+ logger.warn(`[applyTestRunHook] ${hookPath} did not export a function`);
+ return;
+ }
+ hook(run, context);
+ } catch (error) {
+ logger.error(`[applyTestRunHook] Failed to apply hook from ${hookPath}`, error);
+ }
+}