Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

⚡ perf: Async Local File Saves and Compile Hot-Path Regexes Once (#16712)

Marco Beretta8ae65f28dd96
diff --git a/api/app/clients/specs/BaseClient.test.js b/api/app/clients/specs/BaseClient.test.js
index e6578ed3af..cd6ac2c34f 100644
--- a/api/app/clients/specs/BaseClient.test.js
+++ b/api/app/clients/specs/BaseClient.test.js
@@ -3939,12 +3939,13 @@ describe('BaseClient', () => {
/* The stored path is an upload-time inference, so delivery resolves it again by the
* routing settled for the agent running the turn. A test asserting a route has to
* configure that route rather than rely on the stored value alone. */
- const routeTo = (path, ...mimeTypes) => {
+ const routeTo = (path, mimeTypes, endpointConfig = {}) => {
TestClient.options.req = {
config: {
fileConfig: {
endpoints: {
[EModelEndpoint.openAI]: {
+ ...endpointConfig,
defaultLLMDeliveryPath: {
overrides: Object.fromEntries(mimeTypes.map((mime) => [mime, path])),
},
@@ -3960,7 +3961,7 @@ describe('BaseClient', () => {
};
test('keeps a none image in returned files without adding image URLs', async () => {
- routeTo('none', 'image/*');
+ routeTo('none', ['image/*']);
const message = {};
const file = {
user: 'user1',
@@ -3981,7 +3982,7 @@ describe('BaseClient', () => {
});
test('does not inject extracted text after the current provider resolves none', () => {
- routeTo('none', 'application/pdf');
+ routeTo('none', ['application/pdf']);
const file = {
file_id: 'none-pdf',
filename: 'report.pdf',
@@ -3994,10 +3995,7 @@ describe('BaseClient', () => {
});
const routeCsvToTools = ({ textFallbackWithoutTools = true } = {}) => {
- routeTo('none', 'text/csv');
- TestClient.options.req.config.fileConfig.endpoints[
- EModelEndpoint.openAI
- ].textFallbackWithoutTools = textFallbackWithoutTools;
+ routeTo('none', ['text/csv'], { textFallbackWithoutTools });
};
test('injects the text stored for a tool-routed file when this turn runs no reader', () => {
@@ -4196,7 +4194,7 @@ describe('BaseClient', () => {
});
test('does not inject extracted text when the current provider resolves native delivery', () => {
- routeTo('provider', 'application/pdf');
+ routeTo('provider', ['application/pdf']);
const file = {
file_id: 'provider-pdf',
filename: 'report.pdf',
@@ -4216,7 +4214,7 @@ describe('BaseClient', () => {
* extracted text, and extraction at delivery is Phase 2 work. So the model receives
* nothing here either way, which the assertions state rather than imply, and the
* change is limited to not downloading and encoding a file to no purpose. */
- routeTo('text', 'audio/*');
+ routeTo('text', ['audio/*']);
const message = {};
const file = {
user: 'user1',
@@ -4240,7 +4238,7 @@ describe('BaseClient', () => {
test('re-resolves a converted image against the type it was routed on', async () => {
/* Conversion rewrote the stored type, so resolving against that asks about a format
* the administrator never configured a route for and delivers what they excluded. */
- routeTo('none', 'image/png');
+ routeTo('none', ['image/png']);
const message = {};
const file = {
user: 'user1',
@@ -4373,7 +4371,7 @@ describe('BaseClient', () => {
test('keeps an explicitly named destination even under a different provider', async () => {
/* The user named this one, through the chooser or by requesting a tool resource,
* and that decision is not this endpoint's to re-derive. */
- routeTo('text', 'audio/*');
+ routeTo('text', ['audio/*']);
const message = {};
const file = {
user: 'user1',
@@ -4393,7 +4391,7 @@ describe('BaseClient', () => {
});
test('keeps a none PDF in returned files without adding documents', async () => {
- routeTo('none', 'application/pdf');
+ routeTo('none', ['application/pdf']);
const message = {};
const file = {
user: 'user1',
diff --git a/api/server/services/Files/Local/__tests__/crud-traversal.spec.js b/api/server/services/Files/Local/__tests__/crud-traversal.spec.js
index 57ba221d68..8458a59da2 100644
--- a/api/server/services/Files/Local/__tests__/crud-traversal.spec.js
+++ b/api/server/services/Files/Local/__tests__/crud-traversal.spec.js
@@ -1,4 +1,7 @@
-jest.mock('@librechat/api', () => ({ deleteRagFile: jest.fn() }));
+jest.mock('@librechat/api', () => ({
+ deleteRagFile: jest.fn(),
+ saveLocalBuffer: jest.requireActual('@librechat/api').saveLocalBuffer,
+}));
jest.mock('@librechat/data-schemas', () => ({
logger: { warn: jest.fn(), error: jest.fn() },
}));
diff --git a/api/server/services/Files/Local/crud.js b/api/server/services/Files/Local/crud.js
index 9555df49c0..93d3efbb9f 100644
--- a/api/server/services/Files/Local/crud.js
+++ b/api/server/services/Files/Local/crud.js
@@ -3,11 +3,14 @@ const path = require('path');
const axios = require('axios');
const {
deleteRagFile,
+ moveLocalFile,
stripCacheBust,
+ writeLocalFile,
assertRemoteFileURL,
getRemoteFileFetchMaxBytes,
getRemoteFileFetchTimeoutMs,
assertRemoteFileContentLength,
+ saveLocalBuffer: saveBufferToLocalPath,
} = require('@librechat/api');
const { logger } = require('@librechat/data-schemas');
const { EModelEndpoint } = require('librechat-data-provider');
@@ -26,17 +29,8 @@ const paths = require('~/config/paths');
*/
async function saveLocalFile(file, outputPath, outputFilename) {
try {
- if (!fs.existsSync(outputPath)) {
- fs.mkdirSync(outputPath, { recursive: true });
- }
-
const fileExtension = path.extname(file.originalname);
- const filenameWithExt = outputFilename + fileExtension;
- const outputFilePath = path.join(outputPath, filenameWithExt);
- fs.copyFileSync(file.path, outputFilePath);
- fs.unlinkSync(file.path);
-
- return outputFilePath;
+ return await moveLocalFile(file.path, outputPath, outputFilename + fileExtension);
} catch (error) {
logger.error('[saveFile] Error while saving the file:', error);
throw error;
@@ -72,30 +66,7 @@ const saveLocalImage = async (req, file, filename) => {
*/
async function saveLocalBuffer({ userId, buffer, fileName, basePath = 'images' }) {
try {
- const { publicPath, uploads } = paths;
-
- /**
- * For 'images': save to publicPath/images/userId (images are served statically)
- * For 'uploads': save to uploads/userId (files downloaded via API)
- * */
- const directoryPath =
- basePath === 'images' ? path.join(publicPath, basePath, userId) : path.join(uploads, userId);
-
- if (!fs.existsSync(directoryPath)) {
- fs.mkdirSync(directoryPath, { recursive: true });
- }
-
- const resolvedDir = path.resolve(directoryPath);
- const resolvedPath = path.resolve(resolvedDir, fileName);
- const rel = path.relative(resolvedDir, resolvedPath);
- if (rel.startsWith('..') || path.isAbsolute(rel) || rel.includes(`..${path.sep}`)) {
- throw new Error('Path traversal detected in filename');
- }
- fs.writeFileSync(resolvedPath, buffer);
-
- const filePath = path.posix.join('/', basePath, userId, fileName);
-
- return filePath;
+ return await saveBufferToLocalPath({ paths, userId, buffer, fileName, basePath });
} catch (error) {
logger.error('[saveLocalBuffer] Error while saving the buffer:', error);
throw error;
@@ -142,11 +113,6 @@ async function saveFileFromURL({ userId, URL, fileName, basePath = 'images' }) {
// Construct the outputPath based on the basePath and userId
const outputPath = path.join(paths.publicPath, basePath, userId.toString());
- // Check if the output directory exists, if not, create it
- if (!fs.existsSync(outputPath)) {
- fs.mkdirSync(outputPath, { recursive: true });
- }
-
// Replace or append the correct extension
const extRegExp = new RegExp(path.extname(fileName) + '$');
fileName = fileName.replace(extRegExp, `.${extension}`);
@@ -154,9 +120,7 @@ async function saveFileFromURL({ userId, URL, fileName, basePath = 'images' }) {
fileName += `.${extension}`;
}
- // Save the file to the output path
- const outputFilePath = path.join(outputPath, fileName);
- fs.writeFileSync(outputFilePath, buffer);
+ await writeLocalFile(outputPath, fileName, buffer);
return {
bytes,
@@ -304,14 +268,8 @@ async function uploadLocalFile({ req, file, file_id }) {
const { uploads } = appConfig.paths;
const userPath = path.join(uploads, req.user.id);
- if (!fs.existsSync(userPath)) {
- fs.mkdirSync(userPath, { recursive: true });
- }
-
const fileName = `${file_id}__${path.basename(inputFilePath)}`;
- const newPath = path.join(userPath, fileName);
-
- await fs.promises.writeFile(newPath, inputBuffer);
+ const newPath = await writeLocalFile(userPath, fileName, inputBuffer);
const filepath = path.posix.join('/', 'uploads', req.user.id, path.basename(newPath));
let height, width;
diff --git a/api/server/services/Threads/manage.js b/api/server/services/Threads/manage.js
index 2400c72663..f2ae3b152c 100644
--- a/api/server/services/Threads/manage.js
+++ b/api/server/services/Threads/manage.js
@@ -598,6 +598,18 @@ const recordUsage = async ({
const uniqueCitationStart = '^====||===';
const uniqueCitationEnd = '==|||||^';
+const adjacentCitationRegex = new RegExp(
+ `${escapeRegExp(uniqueCitationStart)}(\\d+)${escapeRegExp(
+ uniqueCitationEnd,
+ )}(\\s*)${escapeRegExp(uniqueCitationStart)}(\\d+)${escapeRegExp(uniqueCitationEnd)}`,
+ 'g',
+);
+const remainingAdjacentRegex = new RegExp(
+ `(${escapeRegExp(uniqueCitationStart)}(\\d+)${escapeRegExp(uniqueCitationEnd)})\\s*\\1+`,
+ 'g',
+);
+const citationStartRegex = new RegExp(escapeRegExp(uniqueCitationStart), 'g');
+const citationEndRegex = new RegExp(escapeRegExp(uniqueCitationEnd), 'g');
/**
* Sorts, processes, and flattens messages to a single string.
@@ -723,12 +735,6 @@ async function processMessages({ openai, client, messages = [] }) {
await Promise.all(fileRetrievalPromises);
// Handle adjacent identical citations with the unique format
- const adjacentCitationRegex = new RegExp(
- `${escapeRegExp(uniqueCitationStart)}(\\d+)${escapeRegExp(
- uniqueCitationEnd,
- )}(\\s*)${escapeRegExp(uniqueCitationStart)}(\\d+)${escapeRegExp(uniqueCitationEnd)}`,
- 'g',
- );
text = text.replace(adjacentCitationRegex, (match, num1, space, num2) => {
return num1 === num2
? `${uniqueCitationStart}${num1}${uniqueCitationEnd}`
@@ -736,15 +742,11 @@ async function processMessages({ openai, client, messages = [] }) {
});
// Remove any remaining adjacent identical citations
- const remainingAdjacentRegex = new RegExp(
- `(${escapeRegExp(uniqueCitationStart)}(\\d+)${escapeRegExp(uniqueCitationEnd)})\\s*\\1+`,
- 'g',
- );
text = text.replace(remainingAdjacentRegex, '$1');
// Replace the unique citation format with the final format
- text = text.replace(new RegExp(escapeRegExp(uniqueCitationStart), 'g'), '^');
- text = text.replace(new RegExp(escapeRegExp(uniqueCitationEnd), 'g'), '^');
+ text = text.replace(citationStartRegex, '^');
+ text = text.replace(citationEndRegex, '^');
if (sources.size) {
text += '\n\n';
diff --git a/api/strategies/openIdJwtStrategy.js b/api/strategies/openIdJwtStrategy.js
index ca4edd7829..eb584176bd 100644
--- a/api/strategies/openIdJwtStrategy.js
+++ b/api/strategies/openIdJwtStrategy.js
@@ -57,13 +57,20 @@ const getOpenIdAudienceConfig = () => {
const escapeRegExp = (value) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+const issuerTemplateRegexCache = new Map();
+
const issuerMatchesTemplate = (expectedIssuer, actualIssuer) => {
if (!expectedIssuer.includes('{tenantid}')) {
return false;
}
- const escapedTemplate = expectedIssuer.split('{tenantid}').map(escapeRegExp).join('[^/]+');
- return new RegExp(`^${escapedTemplate}$`).test(actualIssuer);
+ let regex = issuerTemplateRegexCache.get(expectedIssuer);
+ if (!regex) {
+ const escapedTemplate = expectedIssuer.split('{tenantid}').map(escapeRegExp).join('[^/]+');
+ regex = new RegExp(`^${escapedTemplate}$`);
+ issuerTemplateRegexCache.set(expectedIssuer, regex);
+ }
+ return regex.test(actualIssuer);
};
const isOpenIdIssuerAllowed = (payload, openIdConfig) => {
diff --git a/e2e/specs/mock/scenarios/local-file-saves.spec.ts b/e2e/specs/mock/scenarios/local-file-saves.spec.ts
new file mode 100644
index 0000000000..8d64977ab2
--- /dev/null
+++ b/e2e/specs/mock/scenarios/local-file-saves.spec.ts
@@ -0,0 +1,248 @@
+import zlib from 'zlib';
+import { randomUUID } from 'crypto';
+import { expect, test } from '@playwright/test';
+import type { APIRequestContext } from '@playwright/test';
+import type { RequestResult } from '../content-filters.helpers';
+import {
+ deleteConversations,
+ deleteMessagesByConversation,
+ seedConversations,
+ seedMessages,
+} from '../db';
+import { loginAdmin, requestResult } from '../content-filters.helpers';
+import { getPrimaryE2EUser } from '../../../setup/users.mock';
+import { MOCK_ENDPOINTS } from '../helpers';
+
+/**
+ * Guards the async local file save path (uploads written without blocking the
+ * event loop must still land on disk whole and be served byte for byte) and the
+ * artifact edit closing-fence check (a fence longer than the opening one still
+ * closes the code block, so an edit keeps the content and the closing fence).
+ */
+
+const NO_PARENT = '00000000-0000-0000-0000-000000000000';
+const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
+
+type JsonObject = Record<string, unknown>;
+
+function expectSuccess(result: RequestResult, status?: number): void {
+ expect(result.ok, result.text).toBe(true);
+ if (status != null) {
+ expect(result.status, result.text).toBe(status);
+ }
+}
+
+function asObject(value: unknown): JsonObject {
+ return value != null && typeof value === 'object' && !Array.isArray(value)
+ ? (value as JsonObject)
+ : {};
+}
+
+function requireString(value: unknown, label: string): string {
+ if (typeof value !== 'string' || value.length === 0) {
+ throw new Error(`${label} must be a non-empty string`);
+ }
+ return value;
+}
+
+function crc32(buffer: Buffer): number {
+ let crc = 0xffffffff;
+ for (const byte of buffer) {
+ crc ^= byte;
+ for (let bit = 0; bit < 8; bit++) {
+ crc = crc & 1 ? (crc >>> 1) ^ 0xedb88320 : crc >>> 1;
+ }
+ }
+ return (crc ^ 0xffffffff) >>> 0;
+}
+
+function pngChunk(type: string, data: Buffer): Buffer {
+ const body = Buffer.concat([Buffer.from(type, 'ascii'), data]);
+ const length = Buffer.alloc(4);
+ length.writeUInt32BE(data.length);
+ const crc = Buffer.alloc(4);
+ crc.writeUInt32BE(crc32(body));
+ return Buffer.concat([length, body, crc]);
+}
+
+/** A valid RGB PNG with a gradient, so the encoded size is not trivially tiny. */
+function buildPng(size: number): Buffer {
+ const header = Buffer.alloc(13);
+ header.writeUInt32BE(size, 0);
+ header.writeUInt32BE(size, 4);
+ header[8] = 8;
+ header[9] = 2;
+ const stride = size * 3 + 1;
+ const raw = Buffer.alloc(stride * size);
+ for (let y = 0; y < size; y++) {
+ for (let x = 0; x < size; x++) {
+ const offset = y * stride + 1 + x * 3;
+ raw[offset] = (x * 255) / size;
+ raw[offset + 1] = (y * 255) / size;
+ raw[offset + 2] = ((x + y) * 255) / (2 * size);
+ }
+ }
+ return Buffer.concat([
+ PNG_SIGNATURE,
+ pngChunk('IHDR', header),
+ pngChunk('IDAT', zlib.deflateSync(raw)),
+ pngChunk('IEND', Buffer.alloc(0)),
+ ]);
+}
+
+async function deleteUploadedFile(
+ request: APIRequestContext,
+ token: string,
+ file: { file_id: string; filepath: string; source?: string },
+): Promise<void> {
+ const result = await requestResult(request, {
+ path: '/api/files',
+ token,
+ method: 'DELETE',
+ data: { files: [file] },
+ });
+ expect([200, 204], `file cleanup failed: ${result.text}`).toContain(result.status);
+}
+
+test('an uploaded document downloads byte identical @scenario:uploaded-document-downloads-byte-identical', async ({
+ request,
+}) => {
+ const token = await loginAdmin(request);
+ const buffer = Buffer.from(`E2E local save ${randomUUID()}\nsecond line ${randomUUID()}\n`);
+ let uploaded: { file_id: string; filepath: string; source?: string } | undefined;
+ try {
+ const upload = await requestResult(request, {
+ path: '/api/files',
+ token,
+ method: 'POST',
+ multipart: {
+ endpoint: MOCK_ENDPOINTS[0].label,
+ endpointType: 'custom',
+ message_file: 'true',
+ file_id: randomUUID(),
+ file: { name: `local-save-${randomUUID()}.txt`, mimeType: 'text/plain', buffer },
+ },
+ });
+ expectSuccess(upload, 200);
+ const body = asObject(upload.body);
+ uploaded = {
+ file_id: requireString(body.file_id, 'uploaded file id'),
+ filepath: requireString(body.filepath, 'uploaded file path'),
+ ...(typeof body.source === 'string' ? { source: body.source } : {}),
+ };
+ const userId = requireString(body.user, 'uploaded file owner');
+
+ const download = await request.get(
+ `/api/files/download/${encodeURIComponent(userId)}/${encodeURIComponent(uploaded.file_id)}`,
+ { headers: { Authorization: `Bearer ${token}` }, failOnStatusCode: false },
+ );
+ expect(download.status(), await download.text()).toBe(200);
+ const downloaded = await download.body();
+ expect(downloaded.equals(buffer)).toBe(true);
+ } finally {
+ if (uploaded) {
+ await deleteUploadedFile(request, token, uploaded);
+ }
+ }
+});
+
+test('an uploaded image serves the whole image @scenario:uploaded-image-serves-whole-image', async ({
+ request,
+}) => {
+ const token = await loginAdmin(request);
+ let uploaded: { file_id: string; filepath: string; source?: string } | undefined;
+ try {
+ const upload = await requestResult(request, {
+ path: '/api/files/images',
+ token,
+ method: 'POST',
+ multipart: {
+ endpoint: MOCK_ENDPOINTS[0].label,
+ endpointType: 'custom',
+ message_file: 'true',
+ file_id: randomUUID(),
+ width: '96',
+ height: '96',
+ file: {
+ name: `local-save-${randomUUID()}.png`,
+ mimeType: 'image/png',
+ buffer: buildPng(96),
+ },
+ },
+ });
+ expectSuccess(upload, 200);
+ const body = asObject(upload.body);
+ uploaded = {
+ file_id: requireString(body.file_id, 'uploaded image id'),
+ filepath: requireString(body.filepath, 'uploaded image path'),
+ ...(typeof body.source === 'string' ? { source: body.source } : {}),
+ };
+
+ const image = await request.get(uploaded.filepath, {
+ headers: { Authorization: `Bearer ${token}` },
+ failOnStatusCode: false,
+ });
+ expect(image.status()).toBe(200);
+ expect(image.headers()['content-type']).toMatch(/^image\//);
+ const bytes = await image.body();
+ expect(bytes.length).toBeGreaterThan(PNG_SIGNATURE.length);
+ /* imageOutputType defaults to png, which the e2e config does not override. */
+ expect(bytes.subarray(0, PNG_SIGNATURE.length).equals(PNG_SIGNATURE)).toBe(true);
+ if (typeof body.bytes === 'number') {
+ expect(bytes.length).toBe(body.bytes);
+ }
+ } finally {
+ if (uploaded) {
+ await deleteUploadedFile(request, token, uploaded);
+ }
+ }
+});
+
+test('an artifact edit keeps a closing fence longer than the opening one @scenario:artifact-edit-keeps-longer-closing-fence', async ({
+ request,
+}) => {
+ const token = await loginAdmin(request);
+ const { email } = getPrimaryE2EUser();
+ const conversationId = randomUUID();
+ const messageId = randomUUID();
+ const body = ['first line', '~~~', 'edit-target', 'last line'].join('\n');
+ const text = `:::artifact{identifier="fence" type="text/markdown" title="Fence"}\n\`\`\`md\n${body}\n\`\`\`\`\n:::\n`;
+ try {
+ await seedConversations(email, [
+ { conversationId, title: 'E2E closing fence', updatedAt: new Date() },
+ ]);
+ await seedMessages(email, conversationId, [
+ {
+ messageId,
+ parentMessageId: NO_PARENT,
+ text: '',
+ isCreatedByUser: false,
+ sender: 'Assistant',
+ content: [{ type: 'text', text }],
+ },
+ ]);
+
+ const save = await requestResult(request, {
+ path: `/api/messages/artifact/${encodeURIComponent(messageId)}`,
+ token,
+ method: 'POST',
+ data: { index: 0, original: 'edit-target', updated: 'edit-target (edited)' },
+ });
+ expectSuccess(save, 200);
+
+ const read = await requestResult(request, {
+ path: `/api/messages/${conversationId}/${messageId}`,
+ token,
+ });
+ expectSuccess(read, 200);
+ /* The route answers with the matching messages as an array. */
+ const [stored] = read.body as Array<{ content?: Array<{ text?: string }> }>;
+ const saved = stored?.content?.[0]?.text;
+ const expected = text.replace('edit-target', 'edit-target (edited)');
+ expect(saved).toBe(expected);
+ expect(saved).toContain('last line\n````\n:::');
+ } finally {
+ await deleteMessagesByConversation([conversationId]);
+ await deleteConversations([conversationId]);
+ }
+});
diff --git a/packages/api/src/agents/hitl/runtime.ts b/packages/api/src/agents/hitl/runtime.ts
index 94795e2e85..99eec20ae7 100644
--- a/packages/api/src/agents/hitl/runtime.ts
+++ b/packages/api/src/agents/hitl/runtime.ts
@@ -122,25 +122,27 @@ export function buildHITLRunWiring(
registry.register('PreToolUse', { hooks: [withNormalizedEdits(hook)] });
continue;
}
+ let regex: RegExp | null = null;
+ try {
+ regex = new RegExp(matcher);
+ } catch {
+ regex = null;
+ }
+ const compiled = regex;
registry.register('PreToolUse', {
hooks: [
withNormalizedEdits(async (input, signal) => {
- let regex: RegExp;
- try {
- regex = new RegExp(matcher);
- } catch {
+ if (compiled == null) {
return {};
}
- regex.lastIndex = 0;
- if (regex.test(input.toolName)) {
+ if (compiled.test(input.toolName)) {
return hook(input, signal);
}
for (const { name, aliasName } of aliases) {
if (name !== input.toolName) {
continue;
}
- regex.lastIndex = 0;
- if (regex.test(aliasName)) {
+ if (compiled.test(aliasName)) {
return hook(input, signal);
}
}
diff --git a/packages/api/src/artifacts/update.ts b/packages/api/src/artifacts/update.ts
index d9c9582827..ac8fe65a09 100644
--- a/packages/api/src/artifacts/update.ts
+++ b/packages/api/src/artifacts/update.ts
@@ -76,8 +76,16 @@ const getCodeFence = (line: string): CodeFence | null => {
};
const isClosingCodeFence = (line: string, openingFence: CodeFence): boolean => {
- const closePattern = new RegExp(`^\\${openingFence.marker}{${openingFence.length},}\\s*$`);
- return closePattern.test(line.trim());
+ const trimmed = line.trim();
+ if (trimmed.length < openingFence.length) {
+ return false;
+ }
+ for (const char of trimmed) {
+ if (char !== openingFence.marker) {
+ return false;
+ }
+ }
+ return true;
};
const findArtifactClose = (text: string, start: number): ArtifactCloseRange | null => {
diff --git a/packages/api/src/storage/__tests__/local.test.ts b/packages/api/src/storage/__tests__/local.test.ts
new file mode 100644
index 0000000000..c688dd13fe
--- /dev/null
+++ b/packages/api/src/storage/__tests__/local.test.ts
@@ -0,0 +1,127 @@
+import os from 'os';
+import path from 'path';
+import { promises as fs } from 'fs';
+import { moveLocalFile, writeLocalFile, saveLocalBuffer, writeFileAtomic } from '../local';
+
+describe('local storage writes', () => {
+ let dir: string;
+
+ beforeEach(async () => {
+ dir = await fs.mkdtemp(path.join(os.tmpdir(), 'local-storage-'));
+ });
+
+ afterEach(async () => {
+ await fs.rm(dir, { recursive: true, force: true });
+ });
+
+ it('leaves one whole payload when writers of different sizes race on one path', async () => {
+ const target = path.join(dir, 'avatar.png');
+ const large = Buffer.alloc(4 * 1024 * 1024, 'a');
+ const small = Buffer.alloc(1024, 'b');
+
+ await Promise.all([writeFileAtomic(target, large), writeFileAtomic(target, small)]);
+
+ const written = await fs.readFile(target);
+ expect(written.equals(large) || written.equals(small)).toBe(true);
+ expect(await fs.readdir(dir)).toEqual(['avatar.png']);
+ });
+
+ it('removes its temp file and rethrows when the rename fails', async () => {
+ const target = path.join(dir, 'occupied');
+ await fs.mkdir(path.join(target, 'child'), { recursive: true });
+
+ await expect(writeFileAtomic(target, 'data')).rejects.toThrow();
+ expect(await fs.readdir(dir)).toEqual(['occupied']);
+ });
+
+ it('saves a buffer under the user directory and returns its URL path', async () => {
+ const paths = { publicPath: path.join(dir, 'public'), uploads: path.join(dir, 'uploads') };
+
+ const images = await saveLocalBuffer({
+ paths,
+ userId: 'user1',
+ buffer: Buffer.from('png'),
+ fileName: 'a.png',
+ });
+ const uploads = await saveLocalBuffer({
+ paths,
+ userId: 'user1',
+ buffer: Buffer.from('doc'),
+ fileName: 'b.txt',
+ basePath: 'uploads',
+ });
+
+ expect(images).toBe('/images/user1/a.png');
+ expect(uploads).toBe('/uploads/user1/b.txt');
+ expect(await fs.readFile(path.join(paths.publicPath, 'images', 'user1', 'a.png'), 'utf8')).toBe(
+ 'png',
+ );
+ expect(await fs.readFile(path.join(paths.uploads, 'user1', 'b.txt'), 'utf8')).toBe('doc');
+ });
+
+ it('rejects a traversing file name without creating any directory', async () => {
+ const paths = { publicPath: path.join(dir, 'public'), uploads: path.join(dir, 'uploads') };
+
+ await expect(
+ saveLocalBuffer({ paths, userId: 'user1', buffer: Buffer.from('x'), fileName: '../x' }),
+ ).rejects.toThrow('Path traversal detected in filename');
+ expect(await fs.readdir(dir)).toEqual([]);
+ });
+
+ it('leaves one whole file when two uploads move to one path at once', async () => {
+ const large = path.join(dir, 'large.tmp');
+ const small = path.join(dir, 'small.tmp');
+ const largeData = Buffer.alloc(4 * 1024 * 1024, 'a');
+ const smallData = Buffer.alloc(1024, 'b');
+ await fs.writeFile(large, largeData);
+ await fs.writeFile(small, smallData);
+
+ const out = path.join(dir, 'out');
+ await Promise.all([
+ moveLocalFile(large, out, 'image.png'),
+ moveLocalFile(small, out, 'image.png'),
+ ]);
+
+ const written = await fs.readFile(path.join(out, 'image.png'));
+ expect(written.equals(largeData) || written.equals(smallData)).toBe(true);
+ expect(await fs.readdir(out)).toEqual(['image.png']);
+ });
+
+ it('rejects a traversing name in the write and move helpers', async () => {
+ const source = path.join(dir, 'upload.tmp');
+ await fs.writeFile(source, 'content');
+ const target = path.join(dir, 'target');
+
+ await expect(writeLocalFile(target, '../escaped', 'x')).rejects.toThrow(
+ 'Path traversal detected in filename',
+ );
+ await expect(moveLocalFile(source, target, '../escaped')).rejects.toThrow(
+ 'Path traversal detected in filename',
+ );
+ expect(await fs.readdir(dir)).toEqual(['upload.tmp']);
+ });
+
+ it('writes and moves to a name near NAME_MAX', async () => {
+ const longName = `${'n'.repeat(250)}.png`;
+ const source = path.join(dir, 'upload.tmp');
+ await fs.writeFile(source, 'moved');
+
+ await writeLocalFile(path.join(dir, 'written'), longName, 'written');
+ await moveLocalFile(source, path.join(dir, 'moved'), longName);
+
+ expect(await fs.readFile(path.join(dir, 'written', longName), 'utf8')).toBe('written');
+ expect(await fs.readFile(path.join(dir, 'moved', longName), 'utf8')).toBe('moved');
+ expect(await fs.readdir(path.join(dir, 'moved'))).toEqual([longName]);
+ });
+
+ it('moves a temp upload into a new directory', async () => {
+ const source = path.join(dir, 'upload.tmp');
+ await fs.writeFile(source, 'content');
+
+ const target = await moveLocalFile(source, path.join(dir, 'out', 'user1'), 'file.txt');
+
+ expect(target).toBe(path.join(dir, 'out', 'user1', 'file.txt'));
+ expect(await fs.readFile(target, 'utf8')).toBe('content');
+ await expect(fs.access(source)).rejects.toThrow();
+ });
+});
diff --git a/packages/api/src/storage/index.ts b/packages/api/src/storage/index.ts
index 718f3437aa..3639c5fa65 100644
--- a/packages/api/src/storage/index.ts
+++ b/packages/api/src/storage/index.ts
@@ -6,3 +6,4 @@ export * from './avatar';
export * from './metadata';
export * from './url';
export * from './path';
+export * from './local';
diff --git a/packages/api/src/storage/local.ts b/packages/api/src/storage/local.ts
new file mode 100644
index 0000000000..5b9de9acb3
--- /dev/null
+++ b/packages/api/src/storage/local.ts
@@ -0,0 +1,93 @@
+import path from 'path';
+import crypto from 'node:crypto';
+import { mkdir, copyFile, rename, unlink, writeFile } from 'fs/promises';
+
+export interface LocalStoragePaths {
+ publicPath: string;
+ uploads: string;
+}
+
+/**
+ * Fills a sibling temp file and renames it into place, so concurrent writers to one path each
+ * land whole (last rename wins) and readers never see a partly written file. The temp name is
+ * fixed length rather than derived from the target, so a target name near NAME_MAX still fits.
+ */
+async function replaceAtomically(
+ filePath: string,
+ fill: (tempPath: string) => Promise<void>,
+): Promise<void> {
+ const tempPath = path.join(path.dirname(filePath), `.${crypto.randomUUID()}.tmp`);
+ try {
+ await fill(tempPath);
+ await rename(tempPath, filePath);
+ } catch (error) {
+ await unlink(tempPath).catch(() => undefined);
+ throw error;
+ }
+}
+
+export async function writeFileAtomic(filePath: string, data: Buffer | string): Promise<void> {
+ await replaceAtomically(filePath, (tempPath) => writeFile(tempPath, data));
+}
+
+/** Resolves `fileName` inside `directory`, rejecting a name that would escape it. */
+function resolveContained(directory: string, fileName: string): string {
+ const resolvedDir = path.resolve(directory);
+ const resolvedPath = path.resolve(resolvedDir, fileName);
+ const rel = path.relative(resolvedDir, resolvedPath);
+ if (!rel || rel.startsWith('..') || path.isAbsolute(rel) || rel.includes(`..${path.sep}`)) {
+ throw new Error('Path traversal detected in filename');
+ }
+ return resolvedPath;
+}
+
+/** Creates `directory` if needed and writes `data` to `fileName` inside it atomically. */
+export async function writeLocalFile(
+ directory: string,
+ fileName: string,
+ data: Buffer | string,
+): Promise<string> {
+ const filePath = resolveContained(directory, fileName);
+ await mkdir(directory, { recursive: true });
+ await writeFileAtomic(filePath, data);
+ return path.join(directory, fileName);
+}
+
+/** Moves a temp upload into `directory` as `fileName`, creating the directory if needed. */
+export async function moveLocalFile(
+ sourcePath: string,
+ directory: string,
+ fileName: string,
+): Promise<string> {
+ const filePath = resolveContained(directory, fileName);
+ await mkdir(directory, { recursive: true });
+ await replaceAtomically(filePath, (tempPath) => copyFile(sourcePath, tempPath));
+ await unlink(sourcePath);
+ return path.join(directory, fileName);
+}
+
+/**
+ * Saves a buffer under `publicPath/images/userId` (served statically) or `uploads/userId`
+ * (downloaded through the API) and returns its URL path. Rejects a `fileName` that
+ * resolves outside that directory before touching the file system.
+ */
+export async function saveLocalBuffer({
+ paths,
+ userId,
+ buffer,
+ fileName,
+ basePath = 'images',
+}: {
+ paths: LocalStoragePaths;
+ userId: string;
+ buffer: Buffer;
+ fileName: string;
+ basePath?: 'images' | 'uploads';
+}): Promise<string> {
+ const directory =
+ basePath === 'images'
+ ? path.join(paths.publicPath, basePath, userId)
+ : path.join(paths.uploads, userId);
+ await writeLocalFile(directory, fileName, buffer);
+ return path.posix.join('/', basePath, userId, fileName);
+}
diff --git a/packages/data-provider/src/file-config.spec.ts b/packages/data-provider/src/file-config.spec.ts
index 211ef9688e..3acee7e7d8 100644
--- a/packages/data-provider/src/file-config.spec.ts
+++ b/packages/data-provider/src/file-config.spec.ts
@@ -2222,3 +2222,23 @@ it('selects web-search routing without changing stored route selection', () => {
}),
).toBe(false);
});
+
+describe('mergeFileConfig memoization', () => {
+ it('returns the same instance for the same dynamic object and for undefined', () => {
+ const dynamic = fileConfigSchema.parse({ serverFileSizeLimit: 5 });
+ expect(mergeFileConfig(dynamic)).toBe(mergeFileConfig(dynamic));
+ expect(mergeFileConfig(undefined)).toBe(mergeFileConfig(undefined));
+ expect(mergeFileConfig(fileConfigSchema.parse({ serverFileSizeLimit: 5 }))).not.toBe(
+ mergeFileConfig(dynamic),
+ );
+ });
+
+ it('clears the cache when the regex compiler is swapped', () => {
+ const dynamic = fileConfigSchema.parse({ serverFileSizeLimit: 5 });
+ const before = mergeFileConfig(dynamic);
+ const beforeStatic = mergeFileConfig(undefined);
+ setFileConfigRegexCompiler((pattern) => new RegExp(pattern));
+ expect(mergeFileConfig(dynamic)).not.toBe(before);
+ expect(mergeFileConfig(undefined)).not.toBe(beforeStatic);
+ });
+});
diff --git a/packages/data-provider/src/file-config.ts b/packages/data-provider/src/file-config.ts
index 1abe1bd6ab..fadeb3352a 100644
--- a/packages/data-provider/src/file-config.ts
+++ b/packages/data-provider/src/file-config.ts
@@ -702,9 +702,17 @@ export type TFileConfig = z.infer<typeof fileConfigSchema>;
*/
let compileMimeRegex: (pattern: string) => RegexLike = (pattern) => new RegExp(pattern);
+type DynamicFileConfig = z.infer<typeof fileConfigSchema>;
+
+/** Merged results are memoized per `dynamic` object identity; callers must treat them as read-only. */
+let mergedConfigCache = new WeakMap<DynamicFileConfig, FileConfig>();
+let staticMergedConfig: FileConfig | undefined;
+
/** Override the MIME-pattern compiler; the server injects a linear-time engine at startup. */
export const setFileConfigRegexCompiler = (compile: (pattern: string) => RegexLike): void => {
compileMimeRegex = compile;
+ mergedConfigCache = new WeakMap();
+ staticMergedConfig = undefined;
};
/** Returned when every configured pattern fails to compile, so consumers that read an empty
@@ -1220,7 +1228,19 @@ export function getEndpointFileConfig(params: {
return defaultConfig;
}
-export function mergeFileConfig(dynamic: z.infer<typeof fileConfigSchema> | undefined): FileConfig {
+export function mergeFileConfig(dynamic: DynamicFileConfig | undefined): FileConfig {
+ if (!dynamic) {
+ return (staticMergedConfig ??= buildMergedFileConfig(undefined));
+ }
+ let merged = mergedConfigCache.get(dynamic);
+ if (!merged) {
+ merged = buildMergedFileConfig(dynamic);
+ mergedConfigCache.set(dynamic, merged);
+ }
+ return merged;
+}
+
+function buildMergedFileConfig(dynamic: DynamicFileConfig | undefined): FileConfig {
const mergedConfig: FileConfig = {
...fileConfig,
endpoints: {