Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🗂️ refactor: Migrate S3 Storage to TypeScript in packages/api (#11947)
Atef Bellaaja0fed6173c18
diff --git a/api/server/controllers/UserController.js b/api/server/controllers/UserController.jsindex 4a1c9135ab..3702f190db 100644--- a/api/server/controllers/UserController.js+++ b/api/server/controllers/UserController.js@@ -1,6 +1,8 @@const mongoose = require('mongoose');const { logger, webSearchKeys } = require('@librechat/data-schemas');const {+ getNewS3URL,+ needsRefresh,MCPOAuthHandler,MCPTokenStorage,normalizeHttpError,@@ -18,7 +20,6 @@ const { verifyOTPOrBackupCode } = require('~/server/services/twoFactorService');const { verifyEmail, resendVerificationEmail } = require('~/server/services/AuthService');const { getMCPManager, getFlowStateManager, getMCPServersRegistry } = require('~/config');const { invalidateCachedTools } = require('~/server/services/Config/getCachedTools');-const { needsRefresh, getNewS3URL } = require('~/server/services/Files/S3/crud');const { processDeleteRequest } = require('~/server/services/Files/process');const { getAppConfig } = require('~/server/services/Config');const { getSoleOwnedResourceIds } = require('~/server/services/PermissionService');diff --git a/api/server/controllers/UserController.spec.js b/api/server/controllers/UserController.spec.jsindex 6c96f067b7..4a96072062 100644--- a/api/server/controllers/UserController.spec.js+++ b/api/server/controllers/UserController.spec.js@@ -59,7 +59,16 @@ jest.mock('~/server/services/AuthService', () => ({resendVerificationEmail: jest.fn(),}));-jest.mock('~/server/services/Files/S3/crud', () => ({+jest.mock('sharp', () =>+ jest.fn(() => ({+ metadata: jest.fn().mockResolvedValue({}),+ toFormat: jest.fn().mockReturnThis(),+ toBuffer: jest.fn().mockResolvedValue(Buffer.alloc(0)),+ })),+);++jest.mock('@librechat/api', () => ({+ ...jest.requireActual('@librechat/api'),needsRefresh: jest.fn(),getNewS3URL: jest.fn(),}));diff --git a/api/server/controllers/agents/v1.js b/api/server/controllers/agents/v1.jsindex 40d80d571f..b6eb4fc22c 100644--- a/api/server/controllers/agents/v1.js+++ b/api/server/controllers/agents/v1.js@@ -3,6 +3,7 @@ const fs = require('fs').promises;const { nanoid } = require('nanoid');const { logger } = require('@librechat/data-schemas');const {+ refreshS3Url,agentCreateSchema,agentUpdateSchema,refreshListAvatars,@@ -35,7 +36,6 @@ const {const { getStrategyFunctions } = require('~/server/services/Files/strategies');const { resizeAvatar } = require('~/server/services/Files/images/avatar');const { getFileStrategy } = require('~/server/utils/getFileStrategy');-const { refreshS3Url } = require('~/server/services/Files/S3/crud');const { filterFile } = require('~/server/services/Files/process');const { getCachedTools } = require('~/server/services/Config');const { getMCPServersRegistry } = require('~/config');diff --git a/api/server/controllers/agents/v1.spec.js b/api/server/controllers/agents/v1.spec.jsindex 9a8dd0a50a..455cea2e7c 100644--- a/api/server/controllers/agents/v1.spec.js+++ b/api/server/controllers/agents/v1.spec.js@@ -22,7 +22,16 @@ jest.mock('~/server/services/Files/images/avatar', () => ({resizeAvatar: jest.fn(),}));-jest.mock('~/server/services/Files/S3/crud', () => ({+jest.mock('sharp', () =>+ jest.fn(() => ({+ metadata: jest.fn().mockResolvedValue({}),+ toFormat: jest.fn().mockReturnThis(),+ toBuffer: jest.fn().mockResolvedValue(Buffer.alloc(0)),+ })),+);++jest.mock('@librechat/api', () => ({+ ...jest.requireActual('@librechat/api'),refreshS3Url: jest.fn(),}));@@ -73,7 +82,7 @@ const {getResourcePermissionsMap,} = require('~/server/services/PermissionService');-const { refreshS3Url } = require('~/server/services/Files/S3/crud');+const { refreshS3Url } = require('@librechat/api');/*** @type {import('mongoose').Model<import('@librechat/data-schemas').IAgent>}diff --git a/api/server/routes/files/files.agents.test.js b/api/server/routes/files/files.agents.test.jsindex e64be9cf4e..5a01df022d 100644--- a/api/server/routes/files/files.agents.test.js+++ b/api/server/routes/files/files.agents.test.js@@ -39,7 +39,16 @@ jest.mock('~/server/services/Tools/credentials', () => ({loadAuthValues: jest.fn(),}));-jest.mock('~/server/services/Files/S3/crud', () => ({+jest.mock('sharp', () =>+ jest.fn(() => ({+ metadata: jest.fn().mockResolvedValue({}),+ toFormat: jest.fn().mockReturnThis(),+ toBuffer: jest.fn().mockResolvedValue(Buffer.alloc(0)),+ })),+);++jest.mock('@librechat/api', () => ({+ ...jest.requireActual('@librechat/api'),refreshS3FileUrls: jest.fn(),}));diff --git a/api/server/routes/files/files.js b/api/server/routes/files/files.jsindex 5578fc6474..e1b420fb5d 100644--- a/api/server/routes/files/files.js+++ b/api/server/routes/files/files.js@@ -1,8 +1,12 @@const fs = require('fs').promises;const express = require('express');const { EnvVar } = require('@librechat/agents');-const { logger } = require('@librechat/data-schemas');-const { verifyAgentUploadPermission, resolveUploadErrorMessage } = require('@librechat/api');+const { logger, SystemCapabilities } = require('@librechat/data-schemas');+const {+ refreshS3FileUrls,+ resolveUploadErrorMessage,+ verifyAgentUploadPermission,+} = require('@librechat/api');const {Time,isUUID,@@ -14,7 +18,6 @@ const {checkOpenAIStorage,isAssistantsEndpoint,} = require('librechat-data-provider');-const { SystemCapabilities } = require('@librechat/data-schemas');const {filterFile,processFileUpload,@@ -26,7 +29,6 @@ const { getStrategyFunctions } = require('~/server/services/Files/strategies');const { getOpenAIClient } = require('~/server/controllers/assistants/helpers');const { checkPermission } = require('~/server/services/PermissionService');const { loadAuthValues } = require('~/server/services/Tools/credentials');-const { refreshS3FileUrls } = require('~/server/services/Files/S3/crud');const { hasAccessToFilesViaAgent } = require('~/server/services/Files');const { cleanFileName } = require('~/server/utils/files');const { hasCapability } = require('~/server/middleware');diff --git a/api/server/routes/files/files.test.js b/api/server/routes/files/files.test.jsindex 457ebabe92..37cbf68b93 100644--- a/api/server/routes/files/files.test.js+++ b/api/server/routes/files/files.test.js@@ -32,7 +32,16 @@ jest.mock('~/server/services/Tools/credentials', () => ({loadAuthValues: jest.fn(),}));-jest.mock('~/server/services/Files/S3/crud', () => ({+jest.mock('sharp', () =>+ jest.fn(() => ({+ metadata: jest.fn().mockResolvedValue({}),+ toFormat: jest.fn().mockReturnThis(),+ toBuffer: jest.fn().mockResolvedValue(Buffer.alloc(0)),+ })),+);++jest.mock('@librechat/api', () => ({+ ...jest.requireActual('@librechat/api'),refreshS3FileUrls: jest.fn(),}));diff --git a/api/server/services/Files/S3/crud.js b/api/server/services/Files/S3/crud.jsdeleted file mode 100644index c821c0696c..0000000000--- a/api/server/services/Files/S3/crud.js+++ /dev/null@@ -1,556 +0,0 @@-const fs = require('fs');-const fetch = require('node-fetch');-const { logger } = require('@librechat/data-schemas');-const { FileSources } = require('librechat-data-provider');-const { getSignedUrl } = require('@aws-sdk/s3-request-presigner');-const { initializeS3, deleteRagFile, isEnabled } = require('@librechat/api');-const {- PutObjectCommand,- GetObjectCommand,- HeadObjectCommand,- DeleteObjectCommand,-} = require('@aws-sdk/client-s3');--const bucketName = process.env.AWS_BUCKET_NAME;-const defaultBasePath = 'images';-const endpoint = process.env.AWS_ENDPOINT_URL;-const forcePathStyle = isEnabled(process.env.AWS_FORCE_PATH_STYLE);--let s3UrlExpirySeconds = 2 * 60; // 2 minutes-let s3RefreshExpiryMs = null;--if (process.env.S3_URL_EXPIRY_SECONDS !== undefined) {- const parsed = parseInt(process.env.S3_URL_EXPIRY_SECONDS, 10);-- if (!isNaN(parsed) && parsed > 0) {- s3UrlExpirySeconds = Math.min(parsed, 7 * 24 * 60 * 60);- } else {- logger.warn(- `[S3] Invalid S3_URL_EXPIRY_SECONDS value: "${process.env.S3_URL_EXPIRY_SECONDS}". Using 2-minute expiry.`,- );- }-}--if (process.env.S3_REFRESH_EXPIRY_MS !== null && process.env.S3_REFRESH_EXPIRY_MS) {- const parsed = parseInt(process.env.S3_REFRESH_EXPIRY_MS, 10);-- if (!isNaN(parsed) && parsed > 0) {- s3RefreshExpiryMs = parsed;- logger.info(`[S3] Using custom refresh expiry time: ${s3RefreshExpiryMs}ms`);- } else {- logger.warn(- `[S3] Invalid S3_REFRESH_EXPIRY_MS value: "${process.env.S3_REFRESH_EXPIRY_MS}". Using default refresh logic.`,- );- }-}--/**- * Constructs the S3 key based on the base path, user ID, and file name.- */-const getS3Key = (basePath, userId, fileName) => `${basePath}/${userId}/${fileName}`;--/**- * Uploads a buffer to S3 and returns a signed URL.- *- * @param {Object} params- * @param {string} params.userId - The user's unique identifier.- * @param {Buffer} params.buffer - The buffer containing file data.- * @param {string} params.fileName - The file name to use in S3.- * @param {string} [params.basePath='images'] - The base path in the bucket.- * @returns {Promise<string>} Signed URL of the uploaded file.- */-async function saveBufferToS3({ userId, buffer, fileName, basePath = defaultBasePath }) {- const key = getS3Key(basePath, userId, fileName);- const params = { Bucket: bucketName, Key: key, Body: buffer };-- try {- const s3 = initializeS3();- await s3.send(new PutObjectCommand(params));- return await getS3URL({ userId, fileName, basePath });- } catch (error) {- logger.error('[saveBufferToS3] Error uploading buffer to S3:', error.message);- throw error;- }-}--/**- * Retrieves a URL for a file stored in S3.- * Returns a signed URL with expiration time or a proxy URL based on config- *- * @param {Object} params- * @param {string} params.userId - The user's unique identifier.- * @param {string} params.fileName - The file name in S3.- * @param {string} [params.basePath='images'] - The base path in the bucket.- * @param {string} [params.customFilename] - Custom filename for Content-Disposition header (overrides extracted filename).- * @param {string} [params.contentType] - Custom content type for the response.- * @returns {Promise<string>} A URL to access the S3 object- */-async function getS3URL({- userId,- fileName,- basePath = defaultBasePath,- customFilename = null,- contentType = null,-}) {- const key = getS3Key(basePath, userId, fileName);- const params = { Bucket: bucketName, Key: key };-- // Add response headers if specified- if (customFilename) {- params.ResponseContentDisposition = `attachment; filename="${customFilename}"`;- }-- if (contentType) {- params.ResponseContentType = contentType;- }-- try {- const s3 = initializeS3();- return await getSignedUrl(s3, new GetObjectCommand(params), { expiresIn: s3UrlExpirySeconds });- } catch (error) {- logger.error('[getS3URL] Error getting signed URL from S3:', error.message);- throw error;- }-}--/**- * Saves a file from a given URL to S3.- *- * @param {Object} params- * @param {string} params.userId - The user's unique identifier.- * @param {string} params.URL - The source URL of the file.- * @param {string} params.fileName - The file name to use in S3.- * @param {string} [params.basePath='images'] - The base path in the bucket.- * @returns {Promise<string>} Signed URL of the uploaded file.- */-async function saveURLToS3({ userId, URL, fileName, basePath = defaultBasePath }) {- try {- const response = await fetch(URL);- const buffer = await response.buffer();- // Optionally you can call getBufferMetadata(buffer) if needed.- return await saveBufferToS3({ userId, buffer, fileName, basePath });- } catch (error) {- logger.error('[saveURLToS3] Error uploading file from URL to S3:', error.message);- throw error;- }-}--/**- * Deletes a file from S3.- *- * @param {Object} params- * @param {ServerRequest} params.req- * @param {MongoFile} params.file - The file object to delete.- * @returns {Promise<void>}- */-async function deleteFileFromS3(req, file) {- await deleteRagFile({ userId: req.user.id, file });-- const key = extractKeyFromS3Url(file.filepath);- const params = { Bucket: bucketName, Key: key };- if (!key.includes(req.user.id)) {- const message = `[deleteFileFromS3] User ID mismatch: ${req.user.id} vs ${key}`;- logger.error(message);- throw new Error(message);- }-- try {- const s3 = initializeS3();-- try {- const headCommand = new HeadObjectCommand(params);- await s3.send(headCommand);- logger.debug('[deleteFileFromS3] File exists, proceeding with deletion');- } catch (headErr) {- if (headErr.name === 'NotFound') {- logger.warn(`[deleteFileFromS3] File does not exist: ${key}`);- return;- }- }-- const deleteResult = await s3.send(new DeleteObjectCommand(params));- logger.debug('[deleteFileFromS3] Delete command response:', JSON.stringify(deleteResult));- try {- await s3.send(new HeadObjectCommand(params));- logger.error('[deleteFileFromS3] File still exists after deletion!');- } catch (verifyErr) {- if (verifyErr.name === 'NotFound') {- logger.debug(`[deleteFileFromS3] Verified file is deleted: ${key}`);- } else {- logger.error('[deleteFileFromS3] Error verifying deletion:', verifyErr);- }- }-- logger.debug('[deleteFileFromS3] S3 File deletion completed');- } catch (error) {- logger.error(`[deleteFileFromS3] Error deleting file from S3: ${error.message}`);- logger.error(error.stack);-- // If the file is not found, we can safely return.- if (error.code === 'NoSuchKey') {- return;- }- throw error;- }-}--/**- * Uploads a local file to S3 by streaming it directly without loading into memory.- *- * @param {Object} params- * @param {import('express').Request} params.req - The Express request (must include user).- * @param {Express.Multer.File} params.file - The file object from Multer.- * @param {string} params.file_id - Unique file identifier.- * @param {string} [params.basePath='images'] - The base path in the bucket.- * @returns {Promise<{ filepath: string, bytes: number }>}- */-async function uploadFileToS3({ req, file, file_id, basePath = defaultBasePath }) {- try {- const inputFilePath = file.path;- const userId = req.user.id;- const fileName = `${file_id}__${file.originalname}`;- const key = getS3Key(basePath, userId, fileName);-- const stats = await fs.promises.stat(inputFilePath);- const bytes = stats.size;- const fileStream = fs.createReadStream(inputFilePath);-- const s3 = initializeS3();- const uploadParams = {- Bucket: bucketName,- Key: key,- Body: fileStream,- };-- await s3.send(new PutObjectCommand(uploadParams));- const fileURL = await getS3URL({ userId, fileName, basePath });- return { filepath: fileURL, bytes };- } catch (error) {- logger.error('[uploadFileToS3] Error streaming file to S3:', error);- try {- if (file && file.path) {- await fs.promises.unlink(file.path);- }- } catch (unlinkError) {- logger.error(- '[uploadFileToS3] Error deleting temporary file, likely already deleted:',- unlinkError.message,- );- }- throw error;- }-}--/**- * Extracts the S3 key from a URL or returns the key if already properly formatted- *- * @param {string} fileUrlOrKey - The file URL or key- * @returns {string} The S3 key- */-function extractKeyFromS3Url(fileUrlOrKey) {- if (!fileUrlOrKey) {- throw new Error('Invalid input: URL or key is empty');- }-- try {- const url = new URL(fileUrlOrKey);- const hostname = url.hostname;- const pathname = url.pathname.substring(1); // Remove leading slash-- // Explicit path-style with custom endpoint: use endpoint pathname for precise key extraction.- // Handles endpoints with a base path (e.g. https://example.com/storage/).- if (endpoint && forcePathStyle) {- const endpointUrl = new URL(endpoint);- const startPos =- endpointUrl.pathname.length +- (endpointUrl.pathname.endsWith('/') ? 0 : 1) +- bucketName.length +- 1;- const key = url.pathname.substring(startPos);- if (!key) {- logger.warn(- `[extractKeyFromS3Url] Extracted key is empty for endpoint path-style URL: ${fileUrlOrKey}`,- );- } else {- logger.debug(`[extractKeyFromS3Url] fileUrlOrKey: ${fileUrlOrKey}, Extracted key: ${key}`);- }- return key;- }-- if (- hostname === 's3.amazonaws.com' ||- hostname.match(/^s3[-.][a-z0-9-]+\.amazonaws\.com$/) ||- (bucketName && pathname.startsWith(`${bucketName}/`))- ) {- // Path-style: https://s3.amazonaws.com/bucket-name/key or custom endpoint (MinIO, R2, etc.)- // Strip the bucket name (first path segment)- const firstSlashIndex = pathname.indexOf('/');- if (firstSlashIndex > 0) {- const key = pathname.substring(firstSlashIndex + 1);-- if (key === '') {- logger.warn(- `[extractKeyFromS3Url] Extracted key is empty after removing bucket name from URL: ${fileUrlOrKey}`,- );- } else {- logger.debug(- `[extractKeyFromS3Url] fileUrlOrKey: ${fileUrlOrKey}, Extracted key: ${key}`,- );- }-- return key;- } else {- logger.warn(- `[extractKeyFromS3Url] Unable to extract key from path-style URL: ${fileUrlOrKey}`,- );- return '';- }- }-- // Virtual-hosted-style or other: https://bucket-name.s3.amazonaws.com/key- // Just return the pathname without leading slash- logger.debug(`[extractKeyFromS3Url] fileUrlOrKey: ${fileUrlOrKey}, Extracted key: ${pathname}`);- return pathname;- } catch (error) {- if (fileUrlOrKey.startsWith('http://') || fileUrlOrKey.startsWith('https://')) {- logger.error(- `[extractKeyFromS3Url] Error parsing URL: ${fileUrlOrKey}, Error: ${error.message}`,- );- } else {- logger.debug(`[extractKeyFromS3Url] Non-URL input, using fallback: ${fileUrlOrKey}`);- }-- const parts = fileUrlOrKey.split('/');-- if (parts.length >= 3 && !fileUrlOrKey.startsWith('http') && !fileUrlOrKey.startsWith('/')) {- return fileUrlOrKey;- }-- const key = fileUrlOrKey.startsWith('/') ? fileUrlOrKey.substring(1) : fileUrlOrKey;- logger.debug(- `[extractKeyFromS3Url] FALLBACK. fileUrlOrKey: ${fileUrlOrKey}, Extracted key: ${key}`,- );- return key;- }-}--/**- * Retrieves a readable stream for a file stored in S3.- *- * @param {ServerRequest} req - Server request object.- * @param {string} filePath - The S3 key of the file.- * @returns {Promise<NodeJS.ReadableStream>}- */-async function getS3FileStream(_req, filePath) {- try {- const Key = extractKeyFromS3Url(filePath);- const params = { Bucket: bucketName, Key };- const s3 = initializeS3();- const data = await s3.send(new GetObjectCommand(params));- return data.Body; // Returns a Node.js ReadableStream.- } catch (error) {- logger.error('[getS3FileStream] Error retrieving S3 file stream:', error);- throw error;- }-}--/**- * Determines if a signed S3 URL is close to expiration- *- * @param {string} signedUrl - The signed S3 URL- * @param {number} bufferSeconds - Buffer time in seconds- * @returns {boolean} True if the URL needs refreshing- */-function needsRefresh(signedUrl, bufferSeconds) {- try {- // Parse the URL- const url = new URL(signedUrl);-- // Check if it has the signature parameters that indicate it's a signed URL- // X-Amz-Signature is the most reliable indicator for AWS signed URLs- if (!url.searchParams.has('X-Amz-Signature')) {- // Not a signed URL, so no expiration to check (or it's already a proxy URL)- return false;- }-- // Extract the expiration time from the URL- const expiresParam = url.searchParams.get('X-Amz-Expires');- const dateParam = url.searchParams.get('X-Amz-Date');-- if (!expiresParam || !dateParam) {- // Missing expiration information, assume it needs refresh to be safe- return true;- }-- // Parse the AWS date format (YYYYMMDDTHHMMSSZ)- const year = dateParam.substring(0, 4);- const month = dateParam.substring(4, 6);- const day = dateParam.substring(6, 8);- const hour = dateParam.substring(9, 11);- const minute = dateParam.substring(11, 13);- const second = dateParam.substring(13, 15);-- const dateObj = new Date(`${year}-${month}-${day}T${hour}:${minute}:${second}Z`);- const expiresAtDate = new Date(dateObj.getTime() + parseInt(expiresParam) * 1000);-- // Check if it's close to expiration- const now = new Date();-- // If S3_REFRESH_EXPIRY_MS is set, use it to determine if URL is expired- if (s3RefreshExpiryMs !== null) {- const urlCreationTime = dateObj.getTime();- const urlAge = now.getTime() - urlCreationTime;- return urlAge >= s3RefreshExpiryMs;- }-- // Otherwise use the default buffer-based logic- const bufferTime = new Date(now.getTime() + bufferSeconds * 1000);- return expiresAtDate <= bufferTime;- } catch (error) {- logger.error('Error checking URL expiration:', error);- // If we can't determine, assume it needs refresh to be safe- return true;- }-}--/**- * Generates a new URL for an expired S3 URL- * @param {string} currentURL - The current file URL- * @returns {Promise<string | undefined>}- */-async function getNewS3URL(currentURL) {- try {- const s3Key = extractKeyFromS3Url(currentURL);- if (!s3Key) {- return;- }- const keyParts = s3Key.split('/');- if (keyParts.length < 3) {- return;- }-- const basePath = keyParts[0];- const userId = keyParts[1];- const fileName = keyParts.slice(2).join('/');-- return await getS3URL({- userId,- fileName,- basePath,- });- } catch (error) {- logger.error('Error getting new S3 URL:', error);- }-}--/**- * Refreshes S3 URLs for an array of files if they're expired or close to expiring- *- * @param {MongoFile[]} files - Array of file documents- * @param {(files: MongoFile[]) => Promise<void>} batchUpdateFiles - Function to update files in the database- * @param {number} [bufferSeconds=3600] - Buffer time in seconds to check for expiration- * @returns {Promise<MongoFile[]>} The files with refreshed URLs if needed- */-async function refreshS3FileUrls(files, batchUpdateFiles, bufferSeconds = 3600) {- if (!files || !Array.isArray(files) || files.length === 0) {- return files;- }-- const filesToUpdate = [];-- for (let i = 0; i < files.length; i++) {- const file = files[i];- if (!file?.file_id) {- continue;- }- if (file.source !== FileSources.s3) {- continue;- }- if (!file.filepath) {- continue;- }- if (!needsRefresh(file.filepath, bufferSeconds)) {- continue;- }- try {- const newURL = await getNewS3URL(file.filepath);- if (!newURL) {- continue;- }- filesToUpdate.push({- file_id: file.file_id,- filepath: newURL,- });- files[i].filepath = newURL;- } catch (error) {- logger.error(`Error refreshing S3 URL for file ${file.file_id}:`, error);- }- }-- if (filesToUpdate.length > 0) {- await batchUpdateFiles(filesToUpdate);- }-- return files;-}--/**- * Refreshes a single S3 URL if it's expired or close to expiring- *- * @param {{ filepath: string, source: string }} fileObj - Simple file object containing filepath and source- * @param {number} [bufferSeconds=3600] - Buffer time in seconds to check for expiration- * @returns {Promise<string>} The refreshed URL or the original URL if no refresh needed- */-async function refreshS3Url(fileObj, bufferSeconds = 3600) {- if (!fileObj || fileObj.source !== FileSources.s3 || !fileObj.filepath) {- return fileObj?.filepath || '';- }-- if (!needsRefresh(fileObj.filepath, bufferSeconds)) {- return fileObj.filepath;- }-- try {- const s3Key = extractKeyFromS3Url(fileObj.filepath);- if (!s3Key) {- logger.warn(`Unable to extract S3 key from URL: ${fileObj.filepath}`);- return fileObj.filepath;- }-- const keyParts = s3Key.split('/');- if (keyParts.length < 3) {- logger.warn(`Invalid S3 key format: ${s3Key}`);- return fileObj.filepath;- }-- const basePath = keyParts[0];- const userId = keyParts[1];- const fileName = keyParts.slice(2).join('/');-- const newUrl = await getS3URL({- userId,- fileName,- basePath,- });-- logger.debug(`Refreshed S3 URL for key: ${s3Key}`);- return newUrl;- } catch (error) {- logger.error(`Error refreshing S3 URL: ${error.message}`);- return fileObj.filepath;- }-}--module.exports = {- saveBufferToS3,- saveURLToS3,- getS3URL,- deleteFileFromS3,- uploadFileToS3,- getS3FileStream,- refreshS3FileUrls,- refreshS3Url,- needsRefresh,- getNewS3URL,- extractKeyFromS3Url,-};diff --git a/api/server/services/Files/S3/images.js b/api/server/services/Files/S3/images.jsdeleted file mode 100644index 9bdae940c3..0000000000--- a/api/server/services/Files/S3/images.js+++ /dev/null@@ -1,129 +0,0 @@-const fs = require('fs');-const path = require('path');-const sharp = require('sharp');-const { logger } = require('@librechat/data-schemas');-const { resizeImageBuffer } = require('../images/resize');-const { updateUser, updateFile } = require('~/models');-const { saveBufferToS3 } = require('./crud');--const defaultBasePath = 'images';--/**- * Resizes, converts, and uploads an image file to S3.- *- * @param {Object} params- * @param {import('express').Request} params.req - Express request (expects `user` and `appConfig.imageOutputType`).- * @param {Express.Multer.File} params.file - File object from Multer.- * @param {string} params.file_id - Unique file identifier.- * @param {any} params.endpoint - Endpoint identifier used in image processing.- * @param {string} [params.resolution='high'] - Desired image resolution.- * @param {string} [params.basePath='images'] - Base path in the bucket.- * @returns {Promise<{ filepath: string, bytes: number, width: number, height: number }>}- */-async function uploadImageToS3({- req,- file,- file_id,- endpoint,- resolution = 'high',- basePath = defaultBasePath,-}) {- try {- const appConfig = req.config;- const inputFilePath = file.path;- const inputBuffer = await fs.promises.readFile(inputFilePath);- const {- buffer: resizedBuffer,- width,- height,- } = await resizeImageBuffer(inputBuffer, resolution, endpoint);- const extension = path.extname(inputFilePath);- const userId = req.user.id;-- let processedBuffer;- let fileName = `${file_id}__${path.basename(inputFilePath)}`;- const targetExtension = `.${appConfig.imageOutputType}`;-- if (extension.toLowerCase() === targetExtension) {- processedBuffer = resizedBuffer;- } else {- processedBuffer = await sharp(resizedBuffer).toFormat(appConfig.imageOutputType).toBuffer();- fileName = fileName.replace(new RegExp(path.extname(fileName) + '$'), targetExtension);- if (!path.extname(fileName)) {- fileName += targetExtension;- }- }-- const downloadURL = await saveBufferToS3({- userId,- buffer: processedBuffer,- fileName,- basePath,- });- await fs.promises.unlink(inputFilePath);- const bytes = Buffer.byteLength(processedBuffer);- return { filepath: downloadURL, bytes, width, height };- } catch (error) {- logger.error('[uploadImageToS3] Error uploading image to S3:', error.message);- throw error;- }-}--/**- * Updates a file record and returns its signed URL.- *- * @param {import('express').Request} req - Express request.- * @param {Object} file - File metadata.- * @returns {Promise<[Promise<any>, string]>}- */-async function prepareImageURLS3(req, file) {- try {- const updatePromise = updateFile({ file_id: file.file_id });- return Promise.all([updatePromise, file.filepath]);- } catch (error) {- logger.error('[prepareImageURLS3] Error preparing image URL:', error.message);- throw error;- }-}--/**- * Processes a user's avatar image by uploading it to S3 and updating the user's avatar URL if required.- *- * @param {Object} params- * @param {Buffer} params.buffer - Avatar image buffer.- * @param {string} params.userId - User's unique identifier.- * @param {string} params.manual - 'true' or 'false' flag for manual update.- * @param {string} [params.agentId] - Optional agent ID if this is an agent avatar.- * @param {string} [params.basePath='images'] - Base path in the bucket.- * @returns {Promise<string>} Signed URL of the uploaded avatar.- */-async function processS3Avatar({ buffer, userId, manual, agentId, basePath = defaultBasePath }) {- try {- const metadata = await sharp(buffer).metadata();- const extension = metadata.format === 'gif' ? 'gif' : 'png';- const timestamp = new Date().getTime();-- /** Unique filename with timestamp and optional agent ID */- const fileName = agentId- ? `agent-${agentId}-avatar-${timestamp}.${extension}`- : `avatar-${timestamp}.${extension}`;-- const downloadURL = await saveBufferToS3({ userId, buffer, fileName, basePath });-- // Only update user record if this is a user avatar (manual === 'true')- if (manual === 'true' && !agentId) {- await updateUser(userId, { avatar: downloadURL });- }-- return downloadURL;- } catch (error) {- logger.error('[processS3Avatar] Error processing S3 avatar:', error.message);- throw error;- }-}--module.exports = {- uploadImageToS3,- prepareImageURLS3,- processS3Avatar,-};diff --git a/api/server/services/Files/S3/index.js b/api/server/services/Files/S3/index.jsdeleted file mode 100644index 21e2f2ba7d..0000000000--- a/api/server/services/Files/S3/index.js+++ /dev/null@@ -1,7 +0,0 @@-const crud = require('./crud');-const images = require('./images');--module.exports = {- ...crud,- ...images,-};diff --git a/api/server/services/Files/strategies.js b/api/server/services/Files/strategies.jsindex 25341b5715..47b39cb87b 100644--- a/api/server/services/Files/strategies.js+++ b/api/server/services/Files/strategies.js@@ -1,6 +1,13 @@const { FileSources } = require('librechat-data-provider');const {+ getS3URL,+ saveURLToS3,parseDocument,+ uploadFileToS3,+ S3ImageService,+ saveBufferToS3,+ getS3FileStream,+ deleteFileFromS3,uploadMistralOCR,uploadAzureMistralOCR,uploadGoogleVertexMistralOCR,@@ -27,17 +34,18 @@ const {processLocalAvatar,getLocalFileStream,} = require('./Local');-const {- getS3URL,- saveURLToS3,- saveBufferToS3,- getS3FileStream,- uploadImageToS3,- prepareImageURLS3,- deleteFileFromS3,- processS3Avatar,- uploadFileToS3,-} = require('./S3');+const { resizeImageBuffer } = require('./images/resize');+const { updateUser, updateFile } = require('~/models');++const s3ImageService = new S3ImageService({+ resizeImageBuffer,+ updateUser,+ updateFile,+});++const uploadImageToS3 = (params) => s3ImageService.uploadImageToS3(params);+const prepareImageURLS3 = (_req, file) => s3ImageService.prepareImageURL(file);+const processS3Avatar = (params) => s3ImageService.processAvatar(params);const {saveBufferToAzure,saveURLToAzure,diff --git a/api/test/server/services/Files/S3/crud.test.js b/api/test/server/services/Files/S3/crud.test.jsdeleted file mode 100644index d847a82cf0..0000000000--- a/api/test/server/services/Files/S3/crud.test.js+++ /dev/null@@ -1,72 +0,0 @@-const { getS3URL } = require('../../../../../server/services/Files/S3/crud');--// Mock AWS SDK-jest.mock('@aws-sdk/client-s3', () => ({- S3Client: jest.fn(() => ({- send: jest.fn(),- })),- GetObjectCommand: jest.fn(),-}));--jest.mock('@aws-sdk/s3-request-presigner', () => ({- getSignedUrl: jest.fn(),-}));--jest.mock('../../../../../config', () => ({- logger: {- error: jest.fn(),- },-}));--const { getSignedUrl } = require('@aws-sdk/s3-request-presigner');-const { GetObjectCommand } = require('@aws-sdk/client-s3');--describe('S3 crud.js - test only new parameter changes', () => {- beforeEach(() => {- jest.clearAllMocks();- process.env.AWS_BUCKET_NAME = 'test-bucket';- });-- // Test only the new customFilename parameter- it('should include customFilename in response headers when provided', async () => {- getSignedUrl.mockResolvedValue('https://test-presigned-url.com');-- await getS3URL({- userId: 'user123',- fileName: 'test.pdf',- customFilename: 'cleaned_filename.pdf',- });-- // Verify the new ResponseContentDisposition parameter is added to GetObjectCommand- const commandArgs = GetObjectCommand.mock.calls[0][0];- expect(commandArgs.ResponseContentDisposition).toBe(- 'attachment; filename="cleaned_filename.pdf"',- );- });-- // Test only the new contentType parameter- it('should include contentType in response headers when provided', async () => {- getSignedUrl.mockResolvedValue('https://test-presigned-url.com');-- await getS3URL({- userId: 'user123',- fileName: 'test.pdf',- contentType: 'application/pdf',- });-- // Verify the new ResponseContentType parameter is added to GetObjectCommand- const commandArgs = GetObjectCommand.mock.calls[0][0];- expect(commandArgs.ResponseContentType).toBe('application/pdf');- });-- it('should work without new parameters (backward compatibility)', async () => {- getSignedUrl.mockResolvedValue('https://test-presigned-url.com');-- const result = await getS3URL({- userId: 'user123',- fileName: 'test.pdf',- });-- expect(result).toBe('https://test-presigned-url.com');- });-});diff --git a/api/test/services/Files/S3/crud.test.js b/api/test/services/Files/S3/crud.test.jsdeleted file mode 100644index c7b46fba4c..0000000000--- a/api/test/services/Files/S3/crud.test.js+++ /dev/null@@ -1,876 +0,0 @@-const fs = require('fs');-const fetch = require('node-fetch');-const { Readable } = require('stream');-const { FileSources } = require('librechat-data-provider');-const {- PutObjectCommand,- GetObjectCommand,- HeadObjectCommand,- DeleteObjectCommand,-} = require('@aws-sdk/client-s3');-const { getSignedUrl } = require('@aws-sdk/s3-request-presigner');--// Mock dependencies-jest.mock('fs');-jest.mock('node-fetch');-jest.mock('@aws-sdk/s3-request-presigner');-jest.mock('@aws-sdk/client-s3');--jest.mock('@librechat/api', () => ({- initializeS3: jest.fn(),- deleteRagFile: jest.fn().mockResolvedValue(undefined),- isEnabled: jest.fn((val) => val === 'true'),-}));--jest.mock('@librechat/data-schemas', () => ({- logger: {- debug: jest.fn(),- info: jest.fn(),- warn: jest.fn(),- error: jest.fn(),- },-}));--const { initializeS3, deleteRagFile } = require('@librechat/api');-const { logger } = require('@librechat/data-schemas');--// Set env vars before requiring crud so module-level constants pick them up-process.env.AWS_BUCKET_NAME = 'test-bucket';-process.env.S3_URL_EXPIRY_SECONDS = '120';--const {- saveBufferToS3,- saveURLToS3,- getS3URL,- deleteFileFromS3,- uploadFileToS3,- getS3FileStream,- refreshS3FileUrls,- refreshS3Url,- needsRefresh,- getNewS3URL,- extractKeyFromS3Url,-} = require('~/server/services/Files/S3/crud');--describe('S3 CRUD Operations', () => {- let mockS3Client;-- beforeEach(() => {- jest.clearAllMocks();-- // Setup mock S3 client- mockS3Client = {- send: jest.fn(),- };- initializeS3.mockReturnValue(mockS3Client);- });-- afterEach(() => {- delete process.env.S3_URL_EXPIRY_SECONDS;- delete process.env.S3_REFRESH_EXPIRY_MS;- delete process.env.AWS_BUCKET_NAME;- });-- describe('saveBufferToS3', () => {- it('should upload a buffer to S3 and return a signed URL', async () => {- const mockBuffer = Buffer.from('test data');- const mockSignedUrl =- 'https://s3.amazonaws.com/test-bucket/images/user123/test.jpg?signature=abc';-- mockS3Client.send.mockResolvedValue({});- getSignedUrl.mockResolvedValue(mockSignedUrl);-- const result = await saveBufferToS3({- userId: 'user123',- buffer: mockBuffer,- fileName: 'test.jpg',- basePath: 'images',- });-- expect(mockS3Client.send).toHaveBeenCalledWith(expect.any(PutObjectCommand));- expect(result).toBe(mockSignedUrl);- });-- it('should use default basePath if not provided', async () => {- const mockBuffer = Buffer.from('test data');- const mockSignedUrl =- 'https://s3.amazonaws.com/test-bucket/images/user123/test.jpg?signature=abc';-- mockS3Client.send.mockResolvedValue({});- getSignedUrl.mockResolvedValue(mockSignedUrl);-- await saveBufferToS3({- userId: 'user123',- buffer: mockBuffer,- fileName: 'test.jpg',- });-- expect(getSignedUrl).toHaveBeenCalled();- });-- it('should handle S3 upload errors', async () => {- const mockBuffer = Buffer.from('test data');- const error = new Error('S3 upload failed');-- mockS3Client.send.mockRejectedValue(error);-- await expect(- saveBufferToS3({- userId: 'user123',- buffer: mockBuffer,- fileName: 'test.jpg',- }),- ).rejects.toThrow('S3 upload failed');-- expect(logger.error).toHaveBeenCalledWith(- '[saveBufferToS3] Error uploading buffer to S3:',- 'S3 upload failed',- );- });- });-- describe('getS3URL', () => {- it('should return a signed URL for a file', async () => {- const mockSignedUrl =- 'https://s3.amazonaws.com/test-bucket/images/user123/file.pdf?signature=xyz';- getSignedUrl.mockResolvedValue(mockSignedUrl);-- const result = await getS3URL({- userId: 'user123',- fileName: 'file.pdf',- basePath: 'documents',- });-- expect(result).toBe(mockSignedUrl);- expect(getSignedUrl).toHaveBeenCalledWith(- mockS3Client,- expect.any(GetObjectCommand),- expect.objectContaining({ expiresIn: 120 }),- );- });-- it('should add custom filename to Content-Disposition header', async () => {- const mockSignedUrl =- 'https://s3.amazonaws.com/test-bucket/images/user123/file.pdf?signature=xyz';- getSignedUrl.mockResolvedValue(mockSignedUrl);-- await getS3URL({- userId: 'user123',- fileName: 'file.pdf',- customFilename: 'custom-name.pdf',- });-- expect(getSignedUrl).toHaveBeenCalled();- });-- it('should add custom content type', async () => {- const mockSignedUrl =- 'https://s3.amazonaws.com/test-bucket/images/user123/file.pdf?signature=xyz';- getSignedUrl.mockResolvedValue(mockSignedUrl);-- await getS3URL({- userId: 'user123',- fileName: 'file.pdf',- contentType: 'application/pdf',- });-- expect(getSignedUrl).toHaveBeenCalled();- });-- it('should handle errors when getting signed URL', async () => {- const error = new Error('Failed to sign URL');- getSignedUrl.mockRejectedValue(error);-- await expect(- getS3URL({- userId: 'user123',- fileName: 'file.pdf',- }),- ).rejects.toThrow('Failed to sign URL');-- expect(logger.error).toHaveBeenCalledWith(- '[getS3URL] Error getting signed URL from S3:',- 'Failed to sign URL',- );- });- });-- describe('saveURLToS3', () => {- it('should fetch a file from URL and save to S3', async () => {- const mockBuffer = Buffer.from('downloaded data');- const mockResponse = {- buffer: jest.fn().mockResolvedValue(mockBuffer),- };- const mockSignedUrl =- 'https://s3.amazonaws.com/test-bucket/images/user123/downloaded.jpg?signature=abc';-- fetch.mockResolvedValue(mockResponse);- mockS3Client.send.mockResolvedValue({});- getSignedUrl.mockResolvedValue(mockSignedUrl);-- const result = await saveURLToS3({- userId: 'user123',- URL: 'https://example.com/image.jpg',- fileName: 'downloaded.jpg',- });-- expect(fetch).toHaveBeenCalledWith('https://example.com/image.jpg');- expect(mockS3Client.send).toHaveBeenCalled();- expect(result).toBe(mockSignedUrl);- });-- it('should handle fetch errors', async () => {- const error = new Error('Network error');- fetch.mockRejectedValue(error);-- await expect(- saveURLToS3({- userId: 'user123',- URL: 'https://example.com/image.jpg',- fileName: 'downloaded.jpg',- }),- ).rejects.toThrow('Network error');-- expect(logger.error).toHaveBeenCalled();- });- });-- describe('deleteFileFromS3', () => {- const mockReq = {- user: { id: 'user123' },- };-- it('should delete a file from S3', async () => {- const mockFile = {- filepath: 'https://s3.amazonaws.com/test-bucket/images/user123/file.jpg',- file_id: 'file123',- };-- // Mock HeadObject to verify file exists- mockS3Client.send- .mockResolvedValueOnce({}) // First HeadObject - exists- .mockResolvedValueOnce({}) // DeleteObject- .mockRejectedValueOnce({ name: 'NotFound' }); // Second HeadObject - deleted-- await deleteFileFromS3(mockReq, mockFile);-- expect(deleteRagFile).toHaveBeenCalledWith({ userId: 'user123', file: mockFile });- expect(mockS3Client.send).toHaveBeenCalledWith(expect.any(HeadObjectCommand));- expect(mockS3Client.send).toHaveBeenCalledWith(expect.any(DeleteObjectCommand));- });-- it('should handle file not found gracefully', async () => {- const mockFile = {- filepath: 'https://s3.amazonaws.com/test-bucket/images/user123/nonexistent.jpg',- file_id: 'file123',- };-- mockS3Client.send.mockRejectedValue({ name: 'NotFound' });-- await deleteFileFromS3(mockReq, mockFile);-- expect(logger.warn).toHaveBeenCalled();- });-- it('should throw error if user ID does not match', async () => {- const mockFile = {- filepath: 'https://s3.amazonaws.com/test-bucket/images/different-user/file.jpg',- file_id: 'file123',- };-- await expect(deleteFileFromS3(mockReq, mockFile)).rejects.toThrow('User ID mismatch');- expect(logger.error).toHaveBeenCalled();- });-- it('should handle NoSuchKey error', async () => {- const mockFile = {- filepath: 'https://s3.amazonaws.com/test-bucket/images/user123/file.jpg',- file_id: 'file123',- };-- mockS3Client.send- .mockResolvedValueOnce({}) // HeadObject - exists- .mockRejectedValueOnce({ code: 'NoSuchKey' }); // DeleteObject fails-- await deleteFileFromS3(mockReq, mockFile);-- expect(logger.debug).toHaveBeenCalled();- });- });-- describe('uploadFileToS3', () => {- const mockReq = {- user: { id: 'user123' },- };-- it('should upload a file from disk to S3', async () => {- const mockFile = {- path: '/tmp/upload.jpg',- originalname: 'photo.jpg',- };- const mockStats = { size: 1024 };- const mockSignedUrl =- 'https://s3.amazonaws.com/test-bucket/images/user123/file123__photo.jpg?signature=xyz';-- fs.promises = { stat: jest.fn().mockResolvedValue(mockStats) };- fs.createReadStream = jest.fn().mockReturnValue(new Readable());- mockS3Client.send.mockResolvedValue({});- getSignedUrl.mockResolvedValue(mockSignedUrl);-- const result = await uploadFileToS3({- req: mockReq,- file: mockFile,- file_id: 'file123',- basePath: 'images',- });-- expect(result).toEqual({- filepath: mockSignedUrl,- bytes: 1024,- });- expect(fs.createReadStream).toHaveBeenCalledWith('/tmp/upload.jpg');- expect(mockS3Client.send).toHaveBeenCalledWith(expect.any(PutObjectCommand));- });-- it('should handle upload errors and clean up temp file', async () => {- const mockFile = {- path: '/tmp/upload.jpg',- originalname: 'photo.jpg',- };- const error = new Error('Upload failed');-- fs.promises = {- stat: jest.fn().mockResolvedValue({ size: 1024 }),- unlink: jest.fn().mockResolvedValue(),- };- fs.createReadStream = jest.fn().mockReturnValue(new Readable());- mockS3Client.send.mockRejectedValue(error);-- await expect(- uploadFileToS3({- req: mockReq,- file: mockFile,- file_id: 'file123',- }),- ).rejects.toThrow('Upload failed');-- expect(logger.error).toHaveBeenCalledWith(- '[uploadFileToS3] Error streaming file to S3:',- error,- );- });- });-- describe('getS3FileStream', () => {- it('should return a readable stream for a file', async () => {- const mockStream = new Readable();- const mockResponse = { Body: mockStream };-- mockS3Client.send.mockResolvedValue(mockResponse);-- const result = await getS3FileStream(- {},- 'https://s3.amazonaws.com/test-bucket/images/user123/file.pdf',- );-- expect(result).toBe(mockStream);- expect(mockS3Client.send).toHaveBeenCalledWith(expect.any(GetObjectCommand));- });-- it('should handle errors when retrieving stream', async () => {- const error = new Error('Stream error');- mockS3Client.send.mockRejectedValue(error);-- await expect(getS3FileStream({}, 'images/user123/file.pdf')).rejects.toThrow('Stream error');- expect(logger.error).toHaveBeenCalled();- });- });-- describe('needsRefresh', () => {- it('should return false for non-signed URLs', () => {- const url = 'https://example.com/proxy/file.jpg';- const result = needsRefresh(url, 3600);- expect(result).toBe(false);- });-- it('should return true for expired signed URLs', () => {- const now = new Date();- const past = new Date(now.getTime() - 3600 * 1000); // 1 hour ago- const dateStr = past- .toISOString()- .replace(/[-:]/g, '')- .replace(/\.\d{3}/, '');-- const url = `https://s3.amazonaws.com/bucket/key?X-Amz-Signature=abc&X-Amz-Date=${dateStr}&X-Amz-Expires=60`;- const result = needsRefresh(url, 60);- expect(result).toBe(true);- });-- it('should return false for URLs that are not close to expiration', () => {- const now = new Date();- const recent = new Date(now.getTime() - 10 * 1000); // 10 seconds ago- const dateStr = recent- .toISOString()- .replace(/[-:]/g, '')- .replace(/\.\d{3}/, '');-- const url = `https://s3.amazonaws.com/bucket/key?X-Amz-Signature=abc&X-Amz-Date=${dateStr}&X-Amz-Expires=7200`;- const result = needsRefresh(url, 60);- expect(result).toBe(false);- });-- it('should use custom refresh expiry when S3_REFRESH_EXPIRY_MS is set', () => {- process.env.S3_REFRESH_EXPIRY_MS = '30000'; // 30 seconds-- const now = new Date();- const recent = new Date(now.getTime() - 31 * 1000); // 31 seconds ago- const dateStr = recent- .toISOString()- .replace(/[-:]/g, '')- .replace(/\.\d{3}/, '');-- const url = `https://s3.amazonaws.com/bucket/key?X-Amz-Signature=abc&X-Amz-Date=${dateStr}&X-Amz-Expires=7200`;-- // Need to reload the module to pick up the env var change- jest.resetModules();- const { needsRefresh: needsRefreshReloaded } = require('~/server/services/Files/S3/crud');-- const result = needsRefreshReloaded(url, 60);- expect(result).toBe(true);- });-- it('should return true for malformed URLs', () => {- const url = 'not-a-valid-url';- const result = needsRefresh(url, 3600);- expect(result).toBe(true);- });- });-- describe('getNewS3URL', () => {- it('should generate a new URL from an existing S3 URL', async () => {- const currentURL =- 'https://s3.amazonaws.com/test-bucket/images/user123/file.jpg?signature=old';- const newURL = 'https://s3.amazonaws.com/test-bucket/images/user123/file.jpg?signature=new';-- getSignedUrl.mockResolvedValue(newURL);-- const result = await getNewS3URL(currentURL);-- expect(result).toBe(newURL);- expect(getSignedUrl).toHaveBeenCalled();- });-- it('should return undefined for invalid URLs', async () => {- const result = await getNewS3URL('invalid-url');- expect(result).toBeUndefined();- });-- it('should handle errors gracefully', async () => {- const currentURL = 'https://s3.amazonaws.com/test-bucket/images/user123/file.jpg';- getSignedUrl.mockRejectedValue(new Error('Failed'));-- const result = await getNewS3URL(currentURL);-- expect(result).toBeUndefined();- expect(logger.error).toHaveBeenCalledWith('Error getting new S3 URL:', expect.any(Error));- });-- it('should construct GetObjectCommand with correct key (no bucket name duplication)', async () => {- const currentURL =- 'https://s3.amazonaws.com/my-bucket/images/user123/file.jpg?X-Amz-Signature=old';- getSignedUrl.mockResolvedValue(- 'https://s3.amazonaws.com/test-bucket/images/user123/file.jpg?signature=new',- );-- await getNewS3URL(currentURL);-- expect(GetObjectCommand).toHaveBeenCalledWith(- expect.objectContaining({ Key: 'images/user123/file.jpg' }),- );- });- });-- describe('refreshS3FileUrls', () => {- it('should refresh expired URLs for multiple files', async () => {- const now = new Date();- const past = new Date(now.getTime() - 3600 * 1000);- const dateStr = past- .toISOString()- .replace(/[-:]/g, '')- .replace(/\.\d{3}/, '');-- const files = [- {- file_id: 'file1',- source: FileSources.s3,- filepath: `https://s3.amazonaws.com/bucket/images/user123/file1.jpg?X-Amz-Signature=abc&X-Amz-Date=${dateStr}&X-Amz-Expires=60`,- },- {- file_id: 'file2',- source: FileSources.s3,- filepath: `https://s3.amazonaws.com/bucket/images/user123/file2.jpg?X-Amz-Signature=def&X-Amz-Date=${dateStr}&X-Amz-Expires=60`,- },- ];-- const newURL1 = 'https://s3.amazonaws.com/bucket/images/user123/file1.jpg?signature=new1';- const newURL2 = 'https://s3.amazonaws.com/bucket/images/user123/file2.jpg?signature=new2';-- getSignedUrl.mockResolvedValueOnce(newURL1).mockResolvedValueOnce(newURL2);-- const mockBatchUpdate = jest.fn().mockResolvedValue();-- const result = await refreshS3FileUrls(files, mockBatchUpdate, 60);-- expect(result[0].filepath).toBe(newURL1);- expect(result[1].filepath).toBe(newURL2);- expect(mockBatchUpdate).toHaveBeenCalledWith([- { file_id: 'file1', filepath: newURL1 },- { file_id: 'file2', filepath: newURL2 },- ]);- });-- it('should skip non-S3 files', async () => {- const files = [- {- file_id: 'file1',- source: 'local',- filepath: '/local/path/file.jpg',- },- ];-- const mockBatchUpdate = jest.fn();-- const result = await refreshS3FileUrls(files, mockBatchUpdate);-- expect(result).toEqual(files);- expect(mockBatchUpdate).not.toHaveBeenCalled();- });-- it('should handle empty or invalid input', async () => {- const mockBatchUpdate = jest.fn();-- const result1 = await refreshS3FileUrls(null, mockBatchUpdate);- expect(result1).toBe(null);-- const result2 = await refreshS3FileUrls([], mockBatchUpdate);- expect(result2).toEqual([]);-- expect(mockBatchUpdate).not.toHaveBeenCalled();- });-- it('should handle errors for individual files gracefully', async () => {- const now = new Date();- const past = new Date(now.getTime() - 3600 * 1000);- const dateStr = past- .toISOString()- .replace(/[-:]/g, '')- .replace(/\.\d{3}/, '');-- const files = [- {- file_id: 'file1',- source: FileSources.s3,- filepath: `https://s3.amazonaws.com/bucket/images/user123/file1.jpg?X-Amz-Signature=abc&X-Amz-Date=${dateStr}&X-Amz-Expires=60`,- },- ];-- getSignedUrl.mockRejectedValue(new Error('Failed to refresh'));- const mockBatchUpdate = jest.fn();-- await refreshS3FileUrls(files, mockBatchUpdate, 60);-- expect(logger.error).toHaveBeenCalledWith('Error getting new S3 URL:', expect.any(Error));- expect(mockBatchUpdate).not.toHaveBeenCalled();- });- });-- describe('refreshS3Url', () => {- it('should refresh an expired S3 URL', async () => {- const now = new Date();- const past = new Date(now.getTime() - 3600 * 1000);- const dateStr = past- .toISOString()- .replace(/[-:]/g, '')- .replace(/\.\d{3}/, '');-- const fileObj = {- source: FileSources.s3,- filepath: `https://s3.amazonaws.com/bucket/images/user123/file.jpg?X-Amz-Signature=abc&X-Amz-Date=${dateStr}&X-Amz-Expires=60`,- };-- const newURL = 'https://s3.amazonaws.com/bucket/images/user123/file.jpg?signature=new';- getSignedUrl.mockResolvedValue(newURL);-- const result = await refreshS3Url(fileObj, 60);-- expect(result).toBe(newURL);- });-- it('should return original URL if not expired', async () => {- const fileObj = {- source: FileSources.s3,- filepath: 'https://example.com/proxy/file.jpg',- };-- const result = await refreshS3Url(fileObj, 3600);-- expect(result).toBe(fileObj.filepath);- expect(getSignedUrl).not.toHaveBeenCalled();- });-- it('should return empty string for null input', async () => {- const result = await refreshS3Url(null);- expect(result).toBe('');- });-- it('should return original URL for non-S3 files', async () => {- const fileObj = {- source: 'local',- filepath: '/local/path/file.jpg',- };-- const result = await refreshS3Url(fileObj);-- expect(result).toBe(fileObj.filepath);- });-- it('should handle errors and return original URL', async () => {- const now = new Date();- const past = new Date(now.getTime() - 3600 * 1000);- const dateStr = past- .toISOString()- .replace(/[-:]/g, '')- .replace(/\.\d{3}/, '');-- const fileObj = {- source: FileSources.s3,- filepath: `https://s3.amazonaws.com/bucket/images/user123/file.jpg?X-Amz-Signature=abc&X-Amz-Date=${dateStr}&X-Amz-Expires=60`,- };-- getSignedUrl.mockRejectedValue(new Error('Refresh failed'));-- const result = await refreshS3Url(fileObj, 60);-- expect(result).toBe(fileObj.filepath);- expect(logger.error).toHaveBeenCalled();- });- });-- describe('extractKeyFromS3Url', () => {- it('should extract key from a full S3 URL', () => {- const url = 'https://s3.amazonaws.com/test-bucket/images/user123/file.jpg';- const result = extractKeyFromS3Url(url);- expect(result).toBe('images/user123/file.jpg');- });-- it('should extract key from a signed S3 URL with query parameters', () => {- const url =- 'https://s3.amazonaws.com/test-bucket/documents/user456/report.pdf?X-Amz-Signature=abc123&X-Amz-Date=20260107';- const result = extractKeyFromS3Url(url);- expect(result).toBe('documents/user456/report.pdf');- });-- it('should extract key from S3 URL with different domain format', () => {- const url = 'https://test-bucket.s3.amazonaws.com/uploads/user789/image.png';- const result = extractKeyFromS3Url(url);- expect(result).toBe('uploads/user789/image.png');- });-- it('should return key as-is if already properly formatted (3+ parts, no http)', () => {- const key = 'images/user123/file.jpg';- const result = extractKeyFromS3Url(key);- expect(result).toBe('images/user123/file.jpg');- });-- it('should handle key with leading slash by removing it', () => {- const key = '/images/user123/file.jpg';- const result = extractKeyFromS3Url(key);- expect(result).toBe('images/user123/file.jpg');- });-- it('should handle simple key without slashes', () => {- const key = 'simple-file.txt';- const result = extractKeyFromS3Url(key);- expect(result).toBe('simple-file.txt');- });-- it('should handle key with only two parts', () => {- const key = 'folder/file.txt';- const result = extractKeyFromS3Url(key);- expect(result).toBe('folder/file.txt');- });-- it('should throw error for empty input', () => {- expect(() => extractKeyFromS3Url('')).toThrow('Invalid input: URL or key is empty');- });-- it('should throw error for null input', () => {- expect(() => extractKeyFromS3Url(null)).toThrow('Invalid input: URL or key is empty');- });-- it('should throw error for undefined input', () => {- expect(() => extractKeyFromS3Url(undefined)).toThrow('Invalid input: URL or key is empty');- });-- it('should handle URLs with encoded characters', () => {- const url = 'https://s3.amazonaws.com/test-bucket/images/user123/my%20file%20name.jpg';- const result = extractKeyFromS3Url(url);- expect(result).toBe('images/user123/my%20file%20name.jpg');- });-- it('should handle deep nested paths', () => {- const url = 'https://s3.amazonaws.com/bucket/a/b/c/d/e/f/file.jpg';- const result = extractKeyFromS3Url(url);- expect(result).toBe('a/b/c/d/e/f/file.jpg');- });-- it('should log debug message when extracting from URL', () => {- const url = 'https://s3.amazonaws.com/bucket/images/user123/file.jpg';- extractKeyFromS3Url(url);- expect(logger.debug).toHaveBeenCalledWith(- expect.stringContaining('[extractKeyFromS3Url] fileUrlOrKey:'),- );- });-- it('should log fallback debug message for non-URL input', () => {- const key = 'simple-file.txt';- extractKeyFromS3Url(key);- expect(logger.debug).toHaveBeenCalledWith(- expect.stringContaining('[extractKeyFromS3Url] FALLBACK'),- );- });-- it('should handle valid URLs that contain only a bucket', () => {- const url = 'https://s3.amazonaws.com/test-bucket/';- const result = extractKeyFromS3Url(url);- expect(logger.warn).toHaveBeenCalledWith(- expect.stringContaining(- '[extractKeyFromS3Url] Extracted key is empty after removing bucket name from URL: https://s3.amazonaws.com/test-bucket/',- ),- );- expect(result).toBe('');- });-- it('should handle invalid URLs that contain only a bucket', () => {- const url = 'https://s3.amazonaws.com/test-bucket';- const result = extractKeyFromS3Url(url);- expect(logger.warn).toHaveBeenCalledWith(- expect.stringContaining(- '[extractKeyFromS3Url] Unable to extract key from path-style URL: https://s3.amazonaws.com/test-bucket',- ),- );- expect(result).toBe('');- });-- // https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html-- // Path-style requests- // https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#path-style-access- // https://s3.region-code.amazonaws.com/bucket-name/key-name- it('should handle formatted according to Path-style regional endpoint', () => {- const url = 'https://s3.us-west-2.amazonaws.com/amzn-s3-demo-bucket1/dogs/puppy.jpg';- const result = extractKeyFromS3Url(url);- expect(result).toBe('dogs/puppy.jpg');- });-- // virtual host style- // https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#virtual-hosted-style-access- // https://bucket-name.s3.region-code.amazonaws.com/key-name- it('should handle formatted according to Virtual-hosted–style Regional endpoint', () => {- const url = 'https://amzn-s3-demo-bucket1.s3.us-west-2.amazonaws.com/dogs/puppy.png';- const result = extractKeyFromS3Url(url);- expect(result).toBe('dogs/puppy.png');- });-- // Legacy endpoints- // https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#VirtualHostingBackwardsCompatibility-- // s3‐Region- // https://bucket-name.s3-region-code.amazonaws.com- it('should handle formatted according to s3‐Region', () => {- const url = 'https://amzn-s3-demo-bucket1.s3-us-west-2.amazonaws.com/puppy.png';- const result = extractKeyFromS3Url(url);- expect(result).toBe('puppy.png');-- const testcase2 = 'https://amzn-s3-demo-bucket1.s3-us-west-2.amazonaws.com/cats/kitten.png';- const result2 = extractKeyFromS3Url(testcase2);- expect(result2).toBe('cats/kitten.png');- });-- // Legacy global endpoint- // bucket-name.s3.amazonaws.com- it('should handle formatted according to Legacy global endpoint', () => {- const url = 'https://amzn-s3-demo-bucket1.s3.amazonaws.com/dogs/puppy.png';- const result = extractKeyFromS3Url(url);- expect(result).toBe('dogs/puppy.png');- });-- it('should handle malformed URL and log error', () => {- const malformedUrl = 'https://invalid url with spaces.com/key';- const result = extractKeyFromS3Url(malformedUrl);-- expect(logger.error).toHaveBeenCalledWith(- expect.stringContaining('[extractKeyFromS3Url] Error parsing URL:'),- );- expect(logger.error).toHaveBeenCalledWith(expect.stringContaining(malformedUrl));-- expect(result).toBe(malformedUrl);- });-- it('should return empty string for regional path-style URL with only bucket (no key)', () => {- const url = 'https://s3.us-west-2.amazonaws.com/my-bucket';- const result = extractKeyFromS3Url(url);- expect(result).toBe('');- expect(logger.warn).toHaveBeenCalledWith(- expect.stringContaining('[extractKeyFromS3Url] Unable to extract key from path-style URL:'),- );- });-- it('should not log error when given a plain S3 key (non-URL input)', () => {- extractKeyFromS3Url('images/user123/file.jpg');- expect(logger.error).not.toHaveBeenCalled();- });-- it('should strip bucket from custom endpoint URLs (MinIO, R2, etc.) using bucketName', () => {- // bucketName is the module-level const 'test-bucket', set before require at top of file- expect(- extractKeyFromS3Url('https://minio.example.com/test-bucket/images/user123/file.jpg'),- ).toBe('images/user123/file.jpg');- expect(- extractKeyFromS3Url(- 'https://abc123.r2.cloudflarestorage.com/test-bucket/images/user123/avatar.png',- ),- ).toBe('images/user123/avatar.png');- });-- it('should use endpoint base path when AWS_ENDPOINT_URL and AWS_FORCE_PATH_STYLE are set', () => {- process.env.AWS_BUCKET_NAME = 'test-bucket';- process.env.AWS_ENDPOINT_URL = 'https://minio.example.com';- process.env.AWS_FORCE_PATH_STYLE = 'true';- jest.resetModules();- const { extractKeyFromS3Url: fn } = require('~/server/services/Files/S3/crud');-- expect(fn('https://minio.example.com/test-bucket/images/user123/file.jpg')).toBe(- 'images/user123/file.jpg',- );-- delete process.env.AWS_ENDPOINT_URL;- delete process.env.AWS_FORCE_PATH_STYLE;- });-- it('should handle endpoint with a base path', () => {- process.env.AWS_BUCKET_NAME = 'test-bucket';- process.env.AWS_ENDPOINT_URL = 'https://example.com/storage/';- process.env.AWS_FORCE_PATH_STYLE = 'true';- jest.resetModules();- const { extractKeyFromS3Url: fn } = require('~/server/services/Files/S3/crud');-- expect(fn('https://example.com/storage/test-bucket/images/user123/file.jpg')).toBe(- 'images/user123/file.jpg',- );-- delete process.env.AWS_ENDPOINT_URL;- delete process.env.AWS_FORCE_PATH_STYLE;- });- });-});diff --git a/package-lock.json b/package-lock.jsonindex 09b994d719..aad4e24fda 100644--- a/package-lock.json+++ b/package-lock.json@@ -329,44 +329,6 @@"url": "https://github.com/sponsors/panva"}},- "api/node_modules/sharp": {- "version": "0.33.5",- "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.5.tgz",- "integrity": "sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw==",- "hasInstallScript": true,- "dependencies": {- "color": "^4.2.3",- "detect-libc": "^2.0.3",- "semver": "^7.6.3"- },- "engines": {- "node": "^18.17.0 || ^20.3.0 || >=21.0.0"- },- "funding": {- "url": "https://opencollective.com/libvips"- },- "optionalDependencies": {- "@img/sharp-darwin-arm64": "0.33.5",- "@img/sharp-darwin-x64": "0.33.5",- "@img/sharp-libvips-darwin-arm64": "1.0.4",- "@img/sharp-libvips-darwin-x64": "1.0.4",- "@img/sharp-libvips-linux-arm": "1.0.5",- "@img/sharp-libvips-linux-arm64": "1.0.4",- "@img/sharp-libvips-linux-s390x": "1.0.4",- "@img/sharp-libvips-linux-x64": "1.0.4",- "@img/sharp-libvips-linuxmusl-arm64": "1.0.4",- "@img/sharp-libvips-linuxmusl-x64": "1.0.4",- "@img/sharp-linux-arm": "0.33.5",- "@img/sharp-linux-arm64": "0.33.5",- "@img/sharp-linux-s390x": "0.33.5",- "@img/sharp-linux-x64": "0.33.5",- "@img/sharp-linuxmusl-arm64": "0.33.5",- "@img/sharp-linuxmusl-x64": "0.33.5",- "@img/sharp-wasm32": "0.33.5",- "@img/sharp-win32-ia32": "0.33.5",- "@img/sharp-win32-x64": "0.33.5"- }- },"api/node_modules/strtok3": {"version": "10.3.4","resolved": "https://registry.npmjs.org/strtok3/-/strtok3-10.3.4.tgz",@@ -19156,6 +19118,34 @@"@sinonjs/commons": "^3.0.1"}},+ "node_modules/@sinonjs/samsam": {+ "version": "8.0.3",+ "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-8.0.3.tgz",+ "integrity": "sha512-hw6HbX+GyVZzmaYNh82Ecj1vdGZrqVIn/keDTg63IgAwiQPO+xCz99uG6Woqgb4tM0mUiFENKZ4cqd7IX94AXQ==",+ "dev": true,+ "license": "BSD-3-Clause",+ "dependencies": {+ "@sinonjs/commons": "^3.0.1",+ "type-detect": "^4.1.0"+ }+ },+ "node_modules/@sinonjs/samsam/node_modules/type-detect": {+ "version": "4.1.0",+ "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.1.0.tgz",+ "integrity": "sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw==",+ "dev": true,+ "license": "MIT",+ "engines": {+ "node": ">=4"+ }+ },+ "node_modules/@sinonjs/text-encoding": {+ "version": "0.7.3",+ "resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.3.tgz",+ "integrity": "sha512-DE427ROAphMQzU4ENbliGYrBSYPXF+TtLg9S8vzeA+OF4ZKzoDdzfL8sxuMUGS/lgRhM6j1URSk9ghf7Xo1tyA==",+ "dev": true,+ "license": "(Unlicense OR Apache-2.0)"+ },"node_modules/@smithy/abort-controller": {"version": "4.2.12","resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-4.2.12.tgz",@@ -21155,6 +21145,23 @@"@types/send": "*"}},+ "node_modules/@types/sinon": {+ "version": "17.0.4",+ "resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-17.0.4.tgz",+ "integrity": "sha512-RHnIrhfPO3+tJT0s7cFaXGZvsL4bbR3/k7z3P312qMS4JaS2Tk+KiwiLx1S0rQ56ERj00u1/BtdyVd0FY+Pdew==",+ "dev": true,+ "license": "MIT",+ "dependencies": {+ "@types/sinonjs__fake-timers": "*"+ }+ },+ "node_modules/@types/sinonjs__fake-timers": {+ "version": "15.0.1",+ "resolved": "https://registry.npmjs.org/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-15.0.1.tgz",+ "integrity": "sha512-Ko2tjWJq8oozHzHV+reuvS5KYIRAokHnGbDwGh/J64LntgpbuylF74ipEL24HCyRjf9FOlBiBHWBR1RlVKsI1w==",+ "dev": true,+ "license": "MIT"+ },"node_modules/@types/stack-utils": {"version": "2.0.3","resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz",@@ -22444,6 +22451,18 @@"url": "https://github.com/sponsors/ljharb"}},+ "node_modules/aws-sdk-client-mock": {+ "version": "4.1.0",+ "resolved": "https://registry.npmjs.org/aws-sdk-client-mock/-/aws-sdk-client-mock-4.1.0.tgz",+ "integrity": "sha512-h/tOYTkXEsAcV3//6C1/7U4ifSpKyJvb6auveAepqqNJl6TdZaPFEtKjBQNf8UxQdDP850knB2i/whq4zlsxJw==",+ "dev": true,+ "license": "MIT",+ "dependencies": {+ "@types/sinon": "^17.0.3",+ "sinon": "^18.0.1",+ "tslib": "^2.1.0"+ }+ },"node_modules/axe-core": {"version": "4.10.2","resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.10.2.tgz",@@ -31321,6 +31340,13 @@"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==","license": "MIT"},+ "node_modules/just-extend": {+ "version": "6.2.0",+ "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-6.2.0.tgz",+ "integrity": "sha512-cYofQu2Xpom82S6qD778jBDpwvvy39s1l/hrYij2u9AMdQcGRpaBu6kY4mVhuno5kJVi1DAz4aiphA2WI1/OAw==",+ "dev": true,+ "license": "MIT"+ },"node_modules/jwa": {"version": "2.0.1","resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz",@@ -34307,6 +34333,30 @@"resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.1.0.tgz","integrity": "sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ=="},+ "node_modules/nise": {+ "version": "6.1.1",+ "resolved": "https://registry.npmjs.org/nise/-/nise-6.1.1.tgz",+ "integrity": "sha512-aMSAzLVY7LyeM60gvBS423nBmIPP+Wy7St7hsb+8/fc1HmeoHJfLO8CKse4u3BtOZvQLJghYPI2i/1WZrEj5/g==",+ "dev": true,+ "license": "BSD-3-Clause",+ "dependencies": {+ "@sinonjs/commons": "^3.0.1",+ "@sinonjs/fake-timers": "^13.0.1",+ "@sinonjs/text-encoding": "^0.7.3",+ "just-extend": "^6.2.0",+ "path-to-regexp": "^8.1.0"+ }+ },+ "node_modules/nise/node_modules/@sinonjs/fake-timers": {+ "version": "13.0.5",+ "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-13.0.5.tgz",+ "integrity": "sha512-36/hTbH2uaWuGVERyC6da9YwGWnzUZXuPro/F2LfsdOsLnCojz/iSH8MxUt/FD2S5XBSVPhmArFUXcpCQ2Hkiw==",+ "dev": true,+ "license": "BSD-3-Clause",+ "dependencies": {+ "@sinonjs/commons": "^3.0.1"+ }+ },"node_modules/node-domexception": {"version": "1.0.0","resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",@@ -39857,6 +39907,45 @@"url": "https://github.com/sponsors/ljharb"}},+ "node_modules/sharp": {+ "version": "0.33.5",+ "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.5.tgz",+ "integrity": "sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw==",+ "hasInstallScript": true,+ "license": "Apache-2.0",+ "dependencies": {+ "color": "^4.2.3",+ "detect-libc": "^2.0.3",+ "semver": "^7.6.3"+ },+ "engines": {+ "node": "^18.17.0 || ^20.3.0 || >=21.0.0"+ },+ "funding": {+ "url": "https://opencollective.com/libvips"+ },+ "optionalDependencies": {+ "@img/sharp-darwin-arm64": "0.33.5",+ "@img/sharp-darwin-x64": "0.33.5",+ "@img/sharp-libvips-darwin-arm64": "1.0.4",+ "@img/sharp-libvips-darwin-x64": "1.0.4",+ "@img/sharp-libvips-linux-arm": "1.0.5",+ "@img/sharp-libvips-linux-arm64": "1.0.4",+ "@img/sharp-libvips-linux-s390x": "1.0.4",+ "@img/sharp-libvips-linux-x64": "1.0.4",+ "@img/sharp-libvips-linuxmusl-arm64": "1.0.4",+ "@img/sharp-libvips-linuxmusl-x64": "1.0.4",+ "@img/sharp-linux-arm": "0.33.5",+ "@img/sharp-linux-arm64": "0.33.5",+ "@img/sharp-linux-s390x": "0.33.5",+ "@img/sharp-linux-x64": "0.33.5",+ "@img/sharp-linuxmusl-arm64": "0.33.5",+ "@img/sharp-linuxmusl-x64": "0.33.5",+ "@img/sharp-wasm32": "0.33.5",+ "@img/sharp-win32-ia32": "0.33.5",+ "@img/sharp-win32-x64": "0.33.5"+ }+ },"node_modules/shebang-command": {"version": "2.0.0","resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",@@ -39990,6 +40079,45 @@"integrity": "sha512-xMO/8eNREtaROt7tJvWJqHBDTMFN4eiQ5I4JRMuilwfnFcV5W9u7RUkueNkdw0jPqGMX36iCywelS5yilTuOxg==","license": "MIT"},+ "node_modules/sinon": {+ "version": "18.0.1",+ "resolved": "https://registry.npmjs.org/sinon/-/sinon-18.0.1.tgz",+ "integrity": "sha512-a2N2TDY1uGviajJ6r4D1CyRAkzE9NNVlYOV1wX5xQDuAk0ONgzgRl0EjCQuRCPxOwp13ghsMwt9Gdldujs39qw==",+ "dev": true,+ "license": "BSD-3-Clause",+ "dependencies": {+ "@sinonjs/commons": "^3.0.1",+ "@sinonjs/fake-timers": "11.2.2",+ "@sinonjs/samsam": "^8.0.0",+ "diff": "^5.2.0",+ "nise": "^6.0.0",+ "supports-color": "^7"+ },+ "funding": {+ "type": "opencollective",+ "url": "https://opencollective.com/sinon"+ }+ },+ "node_modules/sinon/node_modules/@sinonjs/fake-timers": {+ "version": "11.2.2",+ "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-11.2.2.tgz",+ "integrity": "sha512-G2piCSxQ7oWOxwGSAyFHfPIsyeJGXYtc6mFbnFA+kRXkiEnTl8c/8jul2S329iFBnDI9HGoeWWAZvuvOkZccgw==",+ "dev": true,+ "license": "BSD-3-Clause",+ "dependencies": {+ "@sinonjs/commons": "^3.0.0"+ }+ },+ "node_modules/sinon/node_modules/diff": {+ "version": "5.2.2",+ "resolved": "https://registry.npmjs.org/diff/-/diff-5.2.2.tgz",+ "integrity": "sha512-vtcDfH3TOjP8UekytvnHH1o1P4FcUdt4eQ1Y+Abap1tk/OB2MWQvcwS2ClCd1zuIhc3JKOx6p3kod8Vfys3E+A==",+ "dev": true,+ "license": "BSD-3-Clause",+ "engines": {+ "node": ">=0.3.1"+ }+ },"node_modules/slash": {"version": "4.0.0","resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz",@@ -43831,6 +43959,7 @@"@types/react": "^18.2.18","@types/winston": "^2.4.4","@types/yauzl": "^2.10.3",+ "aws-sdk-client-mock": "^4.1.0","jest": "^30.2.0","jest-junit": "^16.0.0","jszip": "^3.10.1",@@ -43883,6 +44012,7 @@"node-fetch": "2.7.0","pdfjs-dist": "^5.4.624","rate-limit-redis": "^4.2.0",+ "sharp": "^0.33.5","undici": "^7.24.1","yauzl": "^3.2.1","zod": "^3.22.4"diff --git a/packages/api/package.json b/packages/api/package.jsonindex 9ca7f9f865..71bb27a3c4 100644--- a/packages/api/package.json+++ b/packages/api/package.json@@ -25,6 +25,7 @@"test:cache-integration:mcp": "jest --testPathPatterns=\"src/mcp/.*\\.cache_integration\\.spec\\.ts$\" --coverage=false","test:cache-integration:stream": "jest --testPathPatterns=\"src/stream/.*\\.stream_integration\\.spec\\.ts$\" --coverage=false --runInBand --forceExit","test:cache-integration": "npm run test:cache-integration:core && npm run test:cache-integration:cluster && npm run test:cache-integration:mcp && npm run test:cache-integration:stream",+ "test:s3-integration": "jest --testPathPatterns=\"src/storage/s3/.*\\.s3_integration\\.spec\\.ts$\" --coverage=false --runInBand","verify": "npm run test:ci","b:clean": "bun run rimraf dist","b:build": "bun run b:clean && bun run rollup -c --silent --bundleConfigAsCjs",@@ -65,6 +66,7 @@"@types/react": "^18.2.18","@types/winston": "^2.4.4","@types/yauzl": "^2.10.3",+ "aws-sdk-client-mock": "^4.1.0","jest": "^30.2.0","jszip": "^3.10.1","jest-junit": "^16.0.0",@@ -120,6 +122,7 @@"node-fetch": "2.7.0","pdfjs-dist": "^5.4.624","rate-limit-redis": "^4.2.0",+ "sharp": "^0.33.5","undici": "^7.24.1","yauzl": "^3.2.1","zod": "^3.22.4"diff --git a/packages/api/src/cdn/__tests__/s3.test.ts b/packages/api/src/cdn/__tests__/s3.test.tsindex 048c652a45..9a522ecc4f 100644--- a/packages/api/src/cdn/__tests__/s3.test.ts+++ b/packages/api/src/cdn/__tests__/s3.test.ts@@ -101,6 +101,14 @@ describe('initializeS3', () => {);});+ it('should throw when AWS_BUCKET_NAME is not set', async () => {+ delete process.env.AWS_BUCKET_NAME;+ const { initializeS3 } = await load();+ expect(() => initializeS3()).toThrow(+ '[S3] AWS_BUCKET_NAME environment variable is required for S3 operations.',+ );+ });+it('should return the same instance on subsequent calls', async () => {const { MockS3Client, initializeS3 } = await load();const first = initializeS3();diff --git a/packages/api/src/cdn/s3.ts b/packages/api/src/cdn/s3.tsindex f6f8527ce4..c2d0e4d1eb 100644--- a/packages/api/src/cdn/s3.ts+++ b/packages/api/src/cdn/s3.ts@@ -25,6 +25,13 @@ export const initializeS3 = (): S3Client | null => {return null;}+ if (!process.env.AWS_BUCKET_NAME) {+ throw new Error(+ '[S3] AWS_BUCKET_NAME environment variable is required for S3 operations. ' ++ 'Please set this environment variable to enable S3 storage.',+ );+ }+// Read the custom endpoint if provided.const endpoint = process.env.AWS_ENDPOINT_URL;const accessKeyId = process.env.AWS_ACCESS_KEY_ID;diff --git a/packages/api/src/index.ts b/packages/api/src/index.tsindex 687ee7aa49..ef32e7b6b0 100644--- a/packages/api/src/index.ts+++ b/packages/api/src/index.ts@@ -37,6 +37,8 @@ export * from './prompts';export * from './endpoints';/* Files */export * from './files';+/* Storage */+export * from './storage';/* Tools */export * from './tools';/* web search */diff --git a/packages/api/src/storage/index.ts b/packages/api/src/storage/index.tsnew file mode 100644index 0000000000..ebd7bd63a9--- /dev/null+++ b/packages/api/src/storage/index.ts@@ -0,0 +1,2 @@+export * from './s3';+export * from './types';diff --git a/packages/api/src/storage/s3/__tests__/crud.test.ts b/packages/api/src/storage/s3/__tests__/crud.test.tsnew file mode 100644index 0000000000..46e66541ec--- /dev/null+++ b/packages/api/src/storage/s3/__tests__/crud.test.ts@@ -0,0 +1,770 @@+import fs from 'fs';+import { Readable } from 'stream';+import { mockClient } from 'aws-sdk-client-mock';+import { sdkStreamMixin } from '@smithy/util-stream';+import { FileSources } from 'librechat-data-provider';+import {+ S3Client,+ PutObjectCommand,+ GetObjectCommand,+ HeadObjectCommand,+ DeleteObjectCommand,+} from '@aws-sdk/client-s3';+import type { TFile } from 'librechat-data-provider';+import type { S3FileRef } from '~/storage/types';+import type { ServerRequest } from '~/types';++const s3Mock = mockClient(S3Client);++jest.mock('fs', () => ({+ ...jest.requireActual('fs'),+ promises: {+ stat: jest.fn(),+ unlink: jest.fn(),+ },+ createReadStream: jest.fn(),+}));++jest.mock('@aws-sdk/s3-request-presigner', () => ({+ getSignedUrl: jest.fn().mockResolvedValue('https://bucket.s3.amazonaws.com/test-key?signed=true'),+}));++jest.mock('~/files', () => ({+ deleteRagFile: jest.fn().mockResolvedValue(undefined),+}));++jest.mock('@librechat/data-schemas', () => ({+ logger: {+ debug: jest.fn(),+ info: jest.fn(),+ warn: jest.fn(),+ error: jest.fn(),+ },+}));++import { getSignedUrl } from '@aws-sdk/s3-request-presigner';+import { deleteRagFile } from '~/files';+import { logger } from '@librechat/data-schemas';++describe('S3 CRUD', () => {+ let originalEnv: NodeJS.ProcessEnv;++ beforeAll(() => {+ originalEnv = { ...process.env };+ process.env.AWS_REGION = 'us-east-1';+ process.env.AWS_BUCKET_NAME = 'test-bucket';+ process.env.S3_URL_EXPIRY_SECONDS = '120';+ });++ afterAll(() => {+ process.env = originalEnv;+ });++ beforeEach(() => {+ s3Mock.reset();+ s3Mock.on(PutObjectCommand).resolves({});+ s3Mock.on(DeleteObjectCommand).resolves({});++ const stream = new Readable();+ stream.push('test content');+ stream.push(null);+ const sdkStream = sdkStreamMixin(stream);+ s3Mock.on(GetObjectCommand).resolves({ Body: sdkStream });++ jest.clearAllMocks();+ });++ describe('getS3Key', () => {+ it('constructs key from basePath, userId, and fileName', async () => {+ const { getS3Key } = await import('../crud');+ const key = getS3Key('images', 'user123', 'file.png');+ expect(key).toBe('images/user123/file.png');+ });++ it('handles nested file names', async () => {+ const { getS3Key } = await import('../crud');+ const key = getS3Key('files', 'user456', 'folder/subfolder/doc.pdf');+ expect(key).toBe('files/user456/folder/subfolder/doc.pdf');+ });++ it('throws if basePath contains a slash', async () => {+ const { getS3Key } = await import('../crud');+ expect(() => getS3Key('a/b', 'user123', 'file.png')).toThrow(+ '[getS3Key] basePath must not contain slashes: "a/b"',+ );+ });+ });++ describe('saveBufferToS3', () => {+ it('uploads buffer and returns signed URL', async () => {+ const { saveBufferToS3 } = await import('../crud');+ const result = await saveBufferToS3({+ userId: 'user123',+ buffer: Buffer.from('test'),+ fileName: 'test.txt',+ basePath: 'files',+ });+ expect(result).toContain('signed=true');+ expect(s3Mock.commandCalls(PutObjectCommand)).toHaveLength(1);+ });++ it('calls PutObjectCommand with correct parameters', async () => {+ const { saveBufferToS3 } = await import('../crud');+ await saveBufferToS3({+ userId: 'user123',+ buffer: Buffer.from('test content'),+ fileName: 'document.pdf',+ basePath: 'documents',+ });++ const calls = s3Mock.commandCalls(PutObjectCommand);+ expect(calls[0].args[0].input).toEqual({+ Bucket: 'test-bucket',+ Key: 'documents/user123/document.pdf',+ Body: Buffer.from('test content'),+ });+ });++ it('uses default basePath if not provided', async () => {+ const { saveBufferToS3 } = await import('../crud');+ await saveBufferToS3({+ userId: 'user123',+ buffer: Buffer.from('test'),+ fileName: 'test.txt',+ });++ const calls = s3Mock.commandCalls(PutObjectCommand);+ expect(calls[0].args[0].input.Key).toBe('images/user123/test.txt');+ });++ it('handles S3 upload errors', async () => {+ s3Mock.on(PutObjectCommand).rejects(new Error('S3 upload failed'));++ const { saveBufferToS3 } = await import('../crud');+ await expect(+ saveBufferToS3({+ userId: 'user123',+ buffer: Buffer.from('test'),+ fileName: 'test.txt',+ }),+ ).rejects.toThrow('S3 upload failed');++ expect(logger.error).toHaveBeenCalledWith(+ '[saveBufferToS3] Error uploading buffer to S3:',+ 'S3 upload failed',+ );+ });+ });++ describe('getS3URL', () => {+ it('returns signed URL', async () => {+ const { getS3URL } = await import('../crud');+ const result = await getS3URL({+ userId: 'user123',+ fileName: 'test.txt',+ basePath: 'files',+ });+ expect(result).toContain('signed=true');+ });++ it('adds custom filename to Content-Disposition header', async () => {+ const { getS3URL } = await import('../crud');+ await getS3URL({+ userId: 'user123',+ fileName: 'test.pdf',+ customFilename: 'custom-name.pdf',+ });++ expect(getSignedUrl).toHaveBeenCalledWith(+ expect.anything(),+ expect.objectContaining({+ input: expect.objectContaining({+ ResponseContentDisposition: 'attachment; filename="custom-name.pdf"',+ }),+ }),+ expect.anything(),+ );+ });++ it('adds custom content type', async () => {+ const { getS3URL } = await import('../crud');+ await getS3URL({+ userId: 'user123',+ fileName: 'test.pdf',+ contentType: 'application/pdf',+ });++ expect(getSignedUrl).toHaveBeenCalledWith(+ expect.anything(),+ expect.objectContaining({+ input: expect.objectContaining({+ ResponseContentType: 'application/pdf',+ }),+ }),+ expect.anything(),+ );+ });++ it('handles errors when getting signed URL', async () => {+ (getSignedUrl as jest.Mock).mockRejectedValueOnce(new Error('Failed to sign URL'));++ const { getS3URL } = await import('../crud');+ await expect(+ getS3URL({+ userId: 'user123',+ fileName: 'file.pdf',+ }),+ ).rejects.toThrow('Failed to sign URL');++ expect(logger.error).toHaveBeenCalledWith(+ '[getS3URL] Error getting signed URL from S3:',+ 'Failed to sign URL',+ );+ });+ });++ describe('saveURLToS3', () => {+ beforeEach(() => {+ global.fetch = jest.fn().mockResolvedValue({+ ok: true,+ arrayBuffer: jest.fn().mockResolvedValue(new ArrayBuffer(8)),+ }) as unknown as typeof fetch;+ });++ it('fetches file from URL and saves to S3', async () => {+ const { saveURLToS3 } = await import('../crud');+ const result = await saveURLToS3({+ userId: 'user123',+ URL: 'https://example.com/image.jpg',+ fileName: 'downloaded.jpg',+ });++ expect(global.fetch).toHaveBeenCalledWith('https://example.com/image.jpg');+ expect(s3Mock.commandCalls(PutObjectCommand)).toHaveLength(1);+ expect(result).toContain('signed=true');+ });++ it('throws error on non-ok response', async () => {+ (global.fetch as unknown as jest.Mock).mockResolvedValueOnce({+ ok: false,+ status: 404,+ statusText: 'Not Found',+ arrayBuffer: jest.fn().mockResolvedValue(new ArrayBuffer(0)),+ });++ const { saveURLToS3 } = await import('../crud');+ await expect(+ saveURLToS3({+ userId: 'user123',+ URL: 'https://example.com/missing.jpg',+ fileName: 'missing.jpg',+ }),+ ).rejects.toThrow('Failed to fetch URL');+ });++ it('handles fetch errors', async () => {+ (global.fetch as unknown as jest.Mock).mockRejectedValueOnce(new Error('Network error'));++ const { saveURLToS3 } = await import('../crud');+ await expect(+ saveURLToS3({+ userId: 'user123',+ URL: 'https://example.com/image.jpg',+ fileName: 'downloaded.jpg',+ }),+ ).rejects.toThrow('Network error');++ expect(logger.error).toHaveBeenCalled();+ });+ });++ describe('deleteFileFromS3', () => {+ const mockReq = { user: { id: 'user123' } } as ServerRequest;++ it('deletes a file from S3', async () => {+ const mockFile = {+ filepath: 'https://bucket.s3.amazonaws.com/images/user123/file.jpg',+ file_id: 'file123',+ } as TFile;++ s3Mock.on(HeadObjectCommand).resolvesOnce({});++ const { deleteFileFromS3 } = await import('../crud');+ await deleteFileFromS3(mockReq, mockFile);++ expect(deleteRagFile).toHaveBeenCalledWith({ userId: 'user123', file: mockFile });+ expect(s3Mock.commandCalls(HeadObjectCommand)).toHaveLength(1);+ expect(s3Mock.commandCalls(DeleteObjectCommand)).toHaveLength(1);+ });++ it('handles file not found gracefully and cleans up RAG', async () => {+ const mockFile = {+ filepath: 'https://bucket.s3.amazonaws.com/images/user123/nonexistent.jpg',+ file_id: 'file123',+ } as TFile;++ s3Mock.on(HeadObjectCommand).rejects({ name: 'NotFound' });++ const { deleteFileFromS3 } = await import('../crud');+ await deleteFileFromS3(mockReq, mockFile);++ expect(logger.warn).toHaveBeenCalled();+ expect(deleteRagFile).toHaveBeenCalledWith({ userId: 'user123', file: mockFile });+ expect(s3Mock.commandCalls(DeleteObjectCommand)).toHaveLength(0);+ });++ it('throws error if user ID does not match', async () => {+ const mockFile = {+ filepath: 'https://bucket.s3.amazonaws.com/images/different-user/file.jpg',+ file_id: 'file123',+ } as TFile;++ const { deleteFileFromS3 } = await import('../crud');+ await expect(deleteFileFromS3(mockReq, mockFile)).rejects.toThrow('User ID mismatch');+ expect(logger.error).toHaveBeenCalled();+ });++ it('handles NoSuchKey error without calling deleteRagFile', async () => {+ const mockFile = {+ filepath: 'https://bucket.s3.amazonaws.com/images/user123/file.jpg',+ file_id: 'file123',+ } as TFile;++ s3Mock.on(HeadObjectCommand).resolvesOnce({});+ const noSuchKeyError = Object.assign(new Error('NoSuchKey'), { name: 'NoSuchKey' });+ s3Mock.on(DeleteObjectCommand).rejects(noSuchKeyError);++ const { deleteFileFromS3 } = await import('../crud');+ await expect(deleteFileFromS3(mockReq, mockFile)).resolves.toBeUndefined();+ expect(deleteRagFile).not.toHaveBeenCalled();+ });+ });++ describe('uploadFileToS3', () => {+ const mockReq = { user: { id: 'user123' } } as ServerRequest;++ it('uploads a file from disk to S3', async () => {+ const mockFile = {+ path: '/tmp/upload.jpg',+ originalname: 'photo.jpg',+ } as Express.Multer.File;++ (fs.promises.stat as jest.Mock).mockResolvedValue({ size: 1024 });+ (fs.createReadStream as jest.Mock).mockReturnValue(new Readable());++ const { uploadFileToS3 } = await import('../crud');+ const result = await uploadFileToS3({+ req: mockReq,+ file: mockFile,+ file_id: 'file123',+ basePath: 'images',+ });++ expect(result).toEqual({+ filepath: expect.stringContaining('signed=true'),+ bytes: 1024,+ });+ expect(fs.createReadStream).toHaveBeenCalledWith('/tmp/upload.jpg');+ expect(s3Mock.commandCalls(PutObjectCommand)).toHaveLength(1);+ expect(fs.promises.unlink).not.toHaveBeenCalled();+ });++ it('handles upload errors and cleans up temp file', async () => {+ const mockFile = {+ path: '/tmp/upload.jpg',+ originalname: 'photo.jpg',+ } as Express.Multer.File;++ (fs.promises.stat as jest.Mock).mockResolvedValue({ size: 1024 });+ (fs.promises.unlink as jest.Mock).mockResolvedValue(undefined);+ (fs.createReadStream as jest.Mock).mockReturnValue(new Readable());+ s3Mock.on(PutObjectCommand).rejects(new Error('Upload failed'));++ const { uploadFileToS3 } = await import('../crud');+ await expect(+ uploadFileToS3({+ req: mockReq,+ file: mockFile,+ file_id: 'file123',+ }),+ ).rejects.toThrow('Upload failed');++ expect(logger.error).toHaveBeenCalledWith(+ '[uploadFileToS3] Error streaming file to S3:',+ expect.any(Error),+ );+ expect(fs.promises.unlink).toHaveBeenCalledWith('/tmp/upload.jpg');+ });+ });++ describe('getS3FileStream', () => {+ it('returns a readable stream for a file', async () => {+ const { getS3FileStream } = await import('../crud');+ const result = await getS3FileStream(+ {} as ServerRequest,+ 'https://bucket.s3.amazonaws.com/images/user123/file.pdf',+ );++ expect(result).toBeInstanceOf(Readable);+ expect(s3Mock.commandCalls(GetObjectCommand)).toHaveLength(1);+ });++ it('handles errors when retrieving stream', async () => {+ s3Mock.on(GetObjectCommand).rejects(new Error('Stream error'));++ const { getS3FileStream } = await import('../crud');+ await expect(getS3FileStream({} as ServerRequest, 'images/user123/file.pdf')).rejects.toThrow(+ 'Stream error',+ );+ expect(logger.error).toHaveBeenCalled();+ });+ });++ describe('needsRefresh', () => {+ it('returns false for non-signed URLs', async () => {+ const { needsRefresh } = await import('../crud');+ const result = needsRefresh('https://example.com/file.png', 3600);+ expect(result).toBe(false);+ });++ it('returns true when URL is expired', async () => {+ const { needsRefresh } = await import('../crud');+ const pastDate = new Date(Date.now() - 2 * 60 * 60 * 1000);+ const dateStr = pastDate.toISOString().replace(/[-:]/g, '').split('.')[0] + 'Z';+ const url = `https://bucket.s3.amazonaws.com/key?X-Amz-Signature=abc&X-Amz-Date=${dateStr}&X-Amz-Expires=3600`;+ const result = needsRefresh(url, 3600);+ expect(result).toBe(true);+ });++ it('returns false when URL is not close to expiration', async () => {+ const { needsRefresh } = await import('../crud');+ const futureDate = new Date(Date.now() + 10 * 60 * 1000);+ const dateStr = futureDate.toISOString().replace(/[-:]/g, '').split('.')[0] + 'Z';+ const url = `https://bucket.s3.amazonaws.com/key?X-Amz-Signature=abc&X-Amz-Date=${dateStr}&X-Amz-Expires=7200`;+ const result = needsRefresh(url, 60);+ expect(result).toBe(false);+ });++ it('returns true when missing expiration parameters', async () => {+ const { needsRefresh } = await import('../crud');+ const url = 'https://bucket.s3.amazonaws.com/key?X-Amz-Signature=abc';+ const result = needsRefresh(url, 3600);+ expect(result).toBe(true);+ });++ it('returns true for malformed URLs', async () => {+ const { needsRefresh } = await import('../crud');+ const result = needsRefresh('not-a-valid-url', 3600);+ expect(result).toBe(true);+ });+ });++ describe('getNewS3URL', () => {+ it('generates a new URL from an existing S3 URL', async () => {+ const { getNewS3URL } = await import('../crud');+ const result = await getNewS3URL(+ 'https://bucket.s3.amazonaws.com/images/user123/file.jpg?signature=old',+ );++ expect(result).toContain('signed=true');+ });++ it('returns undefined for invalid URLs', async () => {+ const { getNewS3URL } = await import('../crud');+ const result = await getNewS3URL('simple-file.txt');+ expect(result).toBeUndefined();+ });++ it('returns undefined when key has insufficient parts', async () => {+ const { getNewS3URL } = await import('../crud');+ // Key with only 2 parts (basePath/userId but no fileName)+ const result = await getNewS3URL('https://bucket.s3.amazonaws.com/images/user123');+ expect(result).toBeUndefined();+ });+ });++ describe('refreshS3FileUrls', () => {+ it('refreshes expired URLs for multiple files', async () => {+ const { refreshS3FileUrls } = await import('../crud');++ const pastDate = new Date(Date.now() - 2 * 60 * 60 * 1000);+ const dateStr = pastDate.toISOString().replace(/[-:]/g, '').split('.')[0] + 'Z';++ const files = [+ {+ file_id: 'file1',+ source: FileSources.s3,+ filepath: `https://bucket.s3.amazonaws.com/images/user123/file1.jpg?X-Amz-Signature=abc&X-Amz-Date=${dateStr}&X-Amz-Expires=60`,+ },+ {+ file_id: 'file2',+ source: FileSources.s3,+ filepath: `https://bucket.s3.amazonaws.com/images/user456/file2.jpg?X-Amz-Signature=def&X-Amz-Date=${dateStr}&X-Amz-Expires=60`,+ },+ ];++ const mockBatchUpdate = jest.fn().mockResolvedValue(undefined);++ const result = await refreshS3FileUrls(files as TFile[], mockBatchUpdate, 60);++ expect(result[0].filepath).toContain('signed=true');+ expect(result[1].filepath).toContain('signed=true');+ expect(mockBatchUpdate).toHaveBeenCalledWith([+ { file_id: 'file1', filepath: expect.stringContaining('signed=true') },+ { file_id: 'file2', filepath: expect.stringContaining('signed=true') },+ ]);+ });++ it('skips non-S3 files', async () => {+ const { refreshS3FileUrls } = await import('../crud');++ const files = [+ {+ file_id: 'file1',+ source: 'local',+ filepath: '/local/path/file.jpg',+ },+ ];++ const mockBatchUpdate = jest.fn();++ const result = await refreshS3FileUrls(files as TFile[], mockBatchUpdate);++ expect(result).toEqual(files);+ expect(mockBatchUpdate).not.toHaveBeenCalled();+ });++ it('handles empty or invalid input', async () => {+ const { refreshS3FileUrls } = await import('../crud');+ const mockBatchUpdate = jest.fn();++ const result1 = await refreshS3FileUrls(null, mockBatchUpdate);+ expect(result1).toEqual([]);++ const result2 = await refreshS3FileUrls(undefined, mockBatchUpdate);+ expect(result2).toEqual([]);++ const result3 = await refreshS3FileUrls([], mockBatchUpdate);+ expect(result3).toEqual([]);++ expect(mockBatchUpdate).not.toHaveBeenCalled();+ });+ });++ describe('refreshS3Url', () => {+ it('refreshes an expired S3 URL', async () => {+ const { refreshS3Url } = await import('../crud');++ const pastDate = new Date(Date.now() - 2 * 60 * 60 * 1000);+ const dateStr = pastDate.toISOString().replace(/[-:]/g, '').split('.')[0] + 'Z';++ const fileObj: S3FileRef = {+ source: FileSources.s3,+ filepath: `https://bucket.s3.amazonaws.com/images/user123/file.jpg?X-Amz-Signature=abc&X-Amz-Date=${dateStr}&X-Amz-Expires=60`,+ };++ const result = await refreshS3Url(fileObj, 60);++ expect(result).toContain('signed=true');+ });++ it('returns original URL if not expired', async () => {+ const { refreshS3Url } = await import('../crud');++ const fileObj: S3FileRef = {+ source: FileSources.s3,+ filepath: 'https://example.com/proxy/file.jpg',+ };++ const result = await refreshS3Url(fileObj, 3600);++ expect(result).toBe(fileObj.filepath);+ });++ it('returns empty string for null input', async () => {+ const { refreshS3Url } = await import('../crud');+ const result = await refreshS3Url(null as unknown as S3FileRef);+ expect(result).toBe('');+ });++ it('returns original URL for non-S3 files', async () => {+ const { refreshS3Url } = await import('../crud');++ const fileObj: S3FileRef = {+ source: 'local',+ filepath: '/local/path/file.jpg',+ };++ const result = await refreshS3Url(fileObj);++ expect(result).toBe(fileObj.filepath);+ });++ it('handles errors and returns original URL', async () => {+ (getSignedUrl as jest.Mock).mockRejectedValueOnce(new Error('Refresh failed'));++ const { refreshS3Url } = await import('../crud');++ const pastDate = new Date(Date.now() - 2 * 60 * 60 * 1000);+ const dateStr = pastDate.toISOString().replace(/[-:]/g, '').split('.')[0] + 'Z';++ const fileObj: S3FileRef = {+ source: FileSources.s3,+ filepath: `https://bucket.s3.amazonaws.com/images/user123/file.jpg?X-Amz-Signature=abc&X-Amz-Date=${dateStr}&X-Amz-Expires=60`,+ };++ const result = await refreshS3Url(fileObj, 60);++ expect(result).toBe(fileObj.filepath);+ expect(logger.error).toHaveBeenCalled();+ });+ });++ describe('extractKeyFromS3Url', () => {+ it('extracts key from virtual-hosted-style URL', async () => {+ const { extractKeyFromS3Url } = await import('../crud');+ const key = extractKeyFromS3Url('https://bucket.s3.amazonaws.com/images/user123/file.png');+ expect(key).toBe('images/user123/file.png');+ });++ it('returns key as-is when not a URL', async () => {+ const { extractKeyFromS3Url } = await import('../crud');+ const key = extractKeyFromS3Url('images/user123/file.png');+ expect(key).toBe('images/user123/file.png');+ });++ it('throws on empty input', async () => {+ const { extractKeyFromS3Url } = await import('../crud');+ expect(() => extractKeyFromS3Url('')).toThrow('Invalid input: URL or key is empty');+ });++ it('handles URL with query parameters', async () => {+ const { extractKeyFromS3Url } = await import('../crud');+ const key = extractKeyFromS3Url(+ 'https://bucket.s3.amazonaws.com/images/user123/file.png?X-Amz-Signature=abc',+ );+ expect(key).toBe('images/user123/file.png');+ });++ it('extracts key from path-style regional endpoint', async () => {+ const { extractKeyFromS3Url } = await import('../crud');+ const key = extractKeyFromS3Url(+ 'https://s3.us-west-2.amazonaws.com/test-bucket/dogs/puppy.jpg',+ );+ expect(key).toBe('dogs/puppy.jpg');+ });++ it('extracts key from virtual-hosted regional endpoint', async () => {+ const { extractKeyFromS3Url } = await import('../crud');+ const key = extractKeyFromS3Url(+ 'https://test-bucket.s3.us-west-2.amazonaws.com/dogs/puppy.png',+ );+ expect(key).toBe('dogs/puppy.png');+ });++ it('extracts key from legacy s3-region format', async () => {+ const { extractKeyFromS3Url } = await import('../crud');+ const key = extractKeyFromS3Url(+ 'https://test-bucket.s3-us-west-2.amazonaws.com/cats/kitten.png',+ );+ expect(key).toBe('cats/kitten.png');+ });++ it('extracts key from legacy global endpoint', async () => {+ const { extractKeyFromS3Url } = await import('../crud');+ const key = extractKeyFromS3Url('https://test-bucket.s3.amazonaws.com/dogs/puppy.png');+ expect(key).toBe('dogs/puppy.png');+ });++ it('handles key with leading slash by removing it', async () => {+ const { extractKeyFromS3Url } = await import('../crud');+ const key = extractKeyFromS3Url('/images/user123/file.jpg');+ expect(key).toBe('images/user123/file.jpg');+ });++ it('handles simple key without slashes', async () => {+ const { extractKeyFromS3Url } = await import('../crud');+ const key = extractKeyFromS3Url('simple-file.txt');+ expect(key).toBe('simple-file.txt');+ });++ it('handles key with only two parts', async () => {+ const { extractKeyFromS3Url } = await import('../crud');+ const key = extractKeyFromS3Url('folder/file.txt');+ expect(key).toBe('folder/file.txt');+ });++ it('handles URLs with encoded characters', async () => {+ const { extractKeyFromS3Url } = await import('../crud');+ const key = extractKeyFromS3Url(+ 'https://bucket.s3.amazonaws.com/test-bucket/images/user123/my%20file%20name.jpg',+ );+ expect(key).toBe('images/user123/my%20file%20name.jpg');+ });++ it('handles deep nested paths', async () => {+ const { extractKeyFromS3Url } = await import('../crud');+ const key = extractKeyFromS3Url(+ 'https://bucket.s3.amazonaws.com/test-bucket/a/b/c/d/e/f/file.jpg',+ );+ expect(key).toBe('a/b/c/d/e/f/file.jpg');+ });++ it('returns empty string for URL with only bucket (no key)', async () => {+ const { extractKeyFromS3Url } = await import('../crud');+ const key = extractKeyFromS3Url('https://s3.us-west-2.amazonaws.com/my-bucket');+ expect(key).toBe('');+ expect(logger.warn).toHaveBeenCalled();+ });++ it('handles malformed URL and returns input', async () => {+ const { extractKeyFromS3Url } = await import('../crud');+ const malformedUrl = 'https://invalid url with spaces.com/key';+ const result = extractKeyFromS3Url(malformedUrl);++ expect(logger.error).toHaveBeenCalled();+ expect(result).toBe(malformedUrl);+ });++ it('strips bucket from custom endpoint URLs (MinIO, R2)', async () => {+ const { extractKeyFromS3Url } = await import('../crud');+ const key = extractKeyFromS3Url(+ 'https://minio.example.com/test-bucket/images/user123/file.jpg',+ );+ expect(key).toBe('images/user123/file.jpg');+ });+ });++ describe('needsRefresh with S3_REFRESH_EXPIRY_MS set', () => {+ beforeEach(() => {+ process.env.S3_REFRESH_EXPIRY_MS = '60000'; // 1 minute+ jest.resetModules();+ });++ afterEach(() => {+ delete process.env.S3_REFRESH_EXPIRY_MS;+ });++ it('returns true when URL age exceeds S3_REFRESH_EXPIRY_MS', async () => {+ const { needsRefresh } = await import('../crud');+ // URL created 2 minutes ago+ const oldDate = new Date(Date.now() - 2 * 60 * 1000);+ const dateStr = oldDate.toISOString().replace(/[-:]/g, '').split('.')[0] + 'Z';+ const url = `https://bucket.s3.amazonaws.com/key?X-Amz-Signature=abc&X-Amz-Date=${dateStr}&X-Amz-Expires=3600`;++ const result = needsRefresh(url, 60);+ expect(result).toBe(true);+ });++ it('returns false when URL age is under S3_REFRESH_EXPIRY_MS', async () => {+ const { needsRefresh } = await import('../crud');+ // URL created 30 seconds ago+ const recentDate = new Date(Date.now() - 30 * 1000);+ const dateStr = recentDate.toISOString().replace(/[-:]/g, '').split('.')[0] + 'Z';+ const url = `https://bucket.s3.amazonaws.com/key?X-Amz-Signature=abc&X-Amz-Date=${dateStr}&X-Amz-Expires=3600`;++ const result = needsRefresh(url, 60);+ expect(result).toBe(false);+ });+ });+});diff --git a/packages/api/src/storage/s3/__tests__/images.test.ts b/packages/api/src/storage/s3/__tests__/images.test.tsnew file mode 100644index 0000000000..065c73cebd--- /dev/null+++ b/packages/api/src/storage/s3/__tests__/images.test.ts@@ -0,0 +1,182 @@+import fs from 'fs';+import type { S3ImageServiceDeps } from '~/storage/s3/images';+import type { ServerRequest } from '~/types';+import { S3ImageService } from '~/storage/s3/images';+import { saveBufferToS3 } from '~/storage/s3/crud';++jest.mock('fs', () => ({+ ...jest.requireActual('fs'),+ promises: {+ readFile: jest.fn(),+ unlink: jest.fn().mockResolvedValue(undefined),+ },+}));++jest.mock('../crud', () => ({+ saveBufferToS3: jest+ .fn()+ .mockResolvedValue('https://bucket.s3.amazonaws.com/avatar.png?signed=true'),+}));++const mockSaveBufferToS3 = jest.mocked(saveBufferToS3);++jest.mock('sharp', () => {+ return jest.fn(() => ({+ metadata: jest.fn().mockResolvedValue({ format: 'png', width: 100, height: 100 }),+ toFormat: jest.fn().mockReturnThis(),+ toBuffer: jest.fn().mockResolvedValue(Buffer.from('processed')),+ }));+});++describe('S3ImageService', () => {+ let service: S3ImageService;+ let mockDeps: S3ImageServiceDeps;++ beforeEach(() => {+ jest.clearAllMocks();++ mockDeps = {+ resizeImageBuffer: jest.fn().mockResolvedValue({+ buffer: Buffer.from('resized'),+ width: 100,+ height: 100,+ }),+ updateUser: jest.fn().mockResolvedValue(undefined),+ updateFile: jest.fn().mockResolvedValue(undefined),+ };++ service = new S3ImageService(mockDeps);+ });++ describe('processAvatar', () => {+ it('uploads avatar and returns URL', async () => {+ const result = await service.processAvatar({+ buffer: Buffer.from('test'),+ userId: 'user123',+ manual: 'false',+ });++ expect(result).toContain('signed=true');+ });++ it('updates user avatar when manual is true', async () => {+ await service.processAvatar({+ buffer: Buffer.from('test'),+ userId: 'user123',+ manual: 'true',+ });++ expect(mockDeps.updateUser).toHaveBeenCalledWith(+ 'user123',+ expect.objectContaining({ avatar: expect.any(String) }),+ );+ });++ it('does not update user when agentId is provided', async () => {+ await service.processAvatar({+ buffer: Buffer.from('test'),+ userId: 'user123',+ manual: 'true',+ agentId: 'agent456',+ });++ expect(mockDeps.updateUser).not.toHaveBeenCalled();+ });++ it('generates agent avatar filename when agentId provided', async () => {+ await service.processAvatar({+ buffer: Buffer.from('test'),+ userId: 'user123',+ manual: 'false',+ agentId: 'agent456',+ });++ expect(mockSaveBufferToS3).toHaveBeenCalledWith(+ expect.objectContaining({+ fileName: expect.stringContaining('agent-agent456-avatar-'),+ }),+ );+ });+ });++ describe('prepareImageURL', () => {+ it('returns tuple with resolved promise and filepath', async () => {+ const file = { file_id: 'file123', filepath: 'https://example.com/file.png' };+ const result = await service.prepareImageURL(file);++ expect(Array.isArray(result)).toBe(true);+ expect(result[1]).toBe('https://example.com/file.png');+ });++ it('calls updateFile with file_id', async () => {+ const file = { file_id: 'file123', filepath: 'https://example.com/file.png' };+ await service.prepareImageURL(file);++ expect(mockDeps.updateFile).toHaveBeenCalledWith({ file_id: 'file123' });+ });+ });++ describe('constructor', () => {+ it('requires dependencies to be passed', () => {+ const newService = new S3ImageService(mockDeps);+ expect(newService).toBeInstanceOf(S3ImageService);+ });+ });++ describe('uploadImageToS3', () => {+ const mockReq = {+ user: { id: 'user123' },+ config: { imageOutputType: 'webp' },+ } as unknown as ServerRequest;++ it('deletes temp file on early failure (readFile throws)', async () => {+ (fs.promises.readFile as jest.Mock).mockRejectedValueOnce(+ new Error('ENOENT: no such file or directory'),+ );+ (fs.promises.unlink as jest.Mock).mockResolvedValueOnce(undefined);++ await expect(+ service.uploadImageToS3({+ req: mockReq,+ file: { path: '/tmp/input.jpg' } as Express.Multer.File,+ file_id: 'file123',+ endpoint: 'openai',+ }),+ ).rejects.toThrow('ENOENT: no such file or directory');++ expect(fs.promises.unlink).toHaveBeenCalledWith('/tmp/input.jpg');+ });++ it('deletes temp file on resize failure (resizeImageBuffer throws)', async () => {+ (fs.promises.readFile as jest.Mock).mockResolvedValueOnce(Buffer.from('raw'));+ (mockDeps.resizeImageBuffer as jest.Mock).mockRejectedValueOnce(new Error('Resize failed'));+ (fs.promises.unlink as jest.Mock).mockResolvedValueOnce(undefined);++ await expect(+ service.uploadImageToS3({+ req: mockReq,+ file: { path: '/tmp/input.jpg' } as Express.Multer.File,+ file_id: 'file123',+ endpoint: 'openai',+ }),+ ).rejects.toThrow('Resize failed');++ expect(fs.promises.unlink).toHaveBeenCalledWith('/tmp/input.jpg');+ });++ it('deletes temp file on success', async () => {+ (fs.promises.readFile as jest.Mock).mockResolvedValueOnce(Buffer.from('raw'));+ (fs.promises.unlink as jest.Mock).mockResolvedValueOnce(undefined);++ const result = await service.uploadImageToS3({+ req: mockReq,+ file: { path: '/tmp/input.webp' } as Express.Multer.File,+ file_id: 'file123',+ endpoint: 'openai',+ });++ expect(result.filepath).toContain('signed=true');+ expect(fs.promises.unlink).toHaveBeenCalledWith('/tmp/input.webp');+ });+ });+});diff --git a/packages/api/src/storage/s3/__tests__/s3.integration.spec.ts b/packages/api/src/storage/s3/__tests__/s3.integration.spec.tsnew file mode 100644index 0000000000..de80e7409b--- /dev/null+++ b/packages/api/src/storage/s3/__tests__/s3.integration.spec.ts@@ -0,0 +1,529 @@+/**+ * S3 Integration Tests+ *+ * These tests run against a REAL S3 bucket. They are skipped when AWS_TEST_BUCKET_NAME is not set.+ *+ * Run with:+ * AWS_TEST_BUCKET_NAME=my-test-bucket npx jest s3.s3_integration+ *+ * Required env vars:+ * - AWS_TEST_BUCKET_NAME: Dedicated test bucket (gates test execution)+ * - AWS_REGION: Defaults to 'us-east-1'+ * - AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY => to avoid error: A dynamic import callback was invoked without -experimental-vm-modules — the AWS SDK credential provider+ */+import fs from 'fs';+import os from 'os';+import path from 'path';+import { Readable } from 'stream';+import { ListObjectsV2Command, DeleteObjectsCommand } from '@aws-sdk/client-s3';+import type { S3Client } from '@aws-sdk/client-s3';+import type { ServerRequest } from '~/types';++const MINIMAL_PNG = Buffer.from([+ 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52,+ 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x02, 0x00, 0x00, 0x00, 0x90, 0x77, 0x53,+ 0xde, 0x00, 0x00, 0x00, 0x0c, 0x49, 0x44, 0x41, 0x54, 0x08, 0xd7, 0x63, 0xf8, 0xff, 0xff, 0x3f,+ 0x00, 0x05, 0xfe, 0x02, 0xfe, 0xdc, 0xcc, 0x59, 0xe7, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4e,+ 0x44, 0xae, 0x42, 0x60, 0x82,+]);++const TEST_BUCKET = process.env.AWS_TEST_BUCKET_NAME;+const TEST_USER_ID = 'test-user-123';+const TEST_RUN_ID = `integration-test-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;+const TEST_BASE_PATH = TEST_RUN_ID;++async function deleteAllWithPrefix(s3: S3Client, bucket: string, prefix: string): Promise<void> {+ let continuationToken: string | undefined;++ do {+ const listCommand = new ListObjectsV2Command({+ Bucket: bucket,+ Prefix: prefix,+ ContinuationToken: continuationToken,+ });+ const response = await s3.send(listCommand);++ if (response.Contents?.length) {+ const deleteCommand = new DeleteObjectsCommand({+ Bucket: bucket,+ Delete: {+ Objects: response.Contents.filter(+ (obj): obj is typeof obj & { Key: string } => obj.Key !== undefined,+ ).map((obj) => ({ Key: obj.Key })),+ },+ });+ await s3.send(deleteCommand);+ }++ continuationToken = response.IsTruncated ? response.NextContinuationToken : undefined;+ } while (continuationToken);+}++describe('S3 Integration Tests', () => {+ if (!TEST_BUCKET) {+ // eslint-disable-next-line jest/expect-expect+ it.skip('Skipped: AWS_TEST_BUCKET_NAME not configured', () => {});+ return;+ }++ let originalEnv: NodeJS.ProcessEnv;+ let tempDir: string;+ let s3Client: S3Client | null = null;++ beforeAll(async () => {+ originalEnv = { ...process.env };++ // Use dedicated test bucket+ process.env.AWS_BUCKET_NAME = TEST_BUCKET;+ process.env.AWS_REGION = process.env.AWS_REGION || 'us-east-1';++ // Reset modules so the next import picks up the updated env vars.+ // s3Client is retained as a plain instance — it remains valid even though+ // beforeEach/afterEach call resetModules() for per-test isolation.+ jest.resetModules();+ const { initializeS3 } = await import('~/cdn/s3');+ s3Client = initializeS3();+ });++ beforeEach(() => {+ tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 's3-integration-'));+ jest.resetModules();+ });++ afterEach(async () => {+ if (tempDir && fs.existsSync(tempDir)) {+ fs.rmSync(tempDir, { recursive: true, force: true });+ }+ jest.resetModules();+ });++ afterAll(async () => {+ // Clean up all test files from this run+ if (s3Client && TEST_BUCKET) {+ await deleteAllWithPrefix(s3Client, TEST_BUCKET, TEST_RUN_ID);+ }+ process.env = originalEnv;+ jest.resetModules();+ });++ describe('getS3Key', () => {+ it('constructs key from basePath, userId, and fileName', async () => {+ const { getS3Key } = await import('../crud');+ const key = getS3Key(TEST_BASE_PATH, TEST_USER_ID, 'test-file.txt');+ expect(key).toBe(`${TEST_BASE_PATH}/${TEST_USER_ID}/test-file.txt`);+ });++ it('handles nested file names', async () => {+ const { getS3Key } = await import('../crud');+ const key = getS3Key(TEST_BASE_PATH, TEST_USER_ID, 'folder/nested/file.pdf');+ expect(key).toBe(`${TEST_BASE_PATH}/${TEST_USER_ID}/folder/nested/file.pdf`);+ });+ });++ describe('saveBufferToS3 and getS3URL', () => {+ it('uploads buffer and returns signed URL', async () => {+ const { saveBufferToS3 } = await import('../crud');+ const testContent = 'Hello, S3!';+ const buffer = Buffer.from(testContent);+ const fileName = `test-${Date.now()}.txt`;++ const downloadURL = await saveBufferToS3({+ userId: TEST_USER_ID,+ buffer,+ fileName,+ basePath: TEST_BASE_PATH,+ });++ expect(downloadURL).toBeDefined();+ expect(downloadURL).toContain('X-Amz-Signature');+ expect(downloadURL).toContain(fileName);+ });++ it('can get signed URL for existing file', async () => {+ const { saveBufferToS3, getS3URL } = await import('../crud');+ const buffer = Buffer.from('test content for URL');+ const fileName = `url-test-${Date.now()}.txt`;++ await saveBufferToS3({+ userId: TEST_USER_ID,+ buffer,+ fileName,+ basePath: TEST_BASE_PATH,+ });++ const signedUrl = await getS3URL({+ userId: TEST_USER_ID,+ fileName,+ basePath: TEST_BASE_PATH,+ });++ expect(signedUrl).toBeDefined();+ expect(signedUrl).toContain('X-Amz-Signature');+ });++ it('can get signed URL with custom filename and content type', async () => {+ const { saveBufferToS3, getS3URL } = await import('../crud');+ const buffer = Buffer.from('custom headers test');+ const fileName = `headers-test-${Date.now()}.txt`;++ await saveBufferToS3({+ userId: TEST_USER_ID,+ buffer,+ fileName,+ basePath: TEST_BASE_PATH,+ });++ const signedUrl = await getS3URL({+ userId: TEST_USER_ID,+ fileName,+ basePath: TEST_BASE_PATH,+ customFilename: 'download.txt',+ contentType: 'text/plain',+ });++ expect(signedUrl).toContain('response-content-disposition');+ expect(signedUrl).toContain('response-content-type');+ });+ });++ describe('saveURLToS3', () => {+ it('fetches URL content and uploads to S3', async () => {+ const { saveURLToS3 } = await import('../crud');+ const fileName = `url-upload-${Date.now()}.json`;++ const downloadURL = await saveURLToS3({+ userId: TEST_USER_ID,+ URL: 'https://raw.githubusercontent.com/danny-avila/LibreChat/main/package.json',+ fileName,+ basePath: TEST_BASE_PATH,+ });++ expect(downloadURL).toBeDefined();+ expect(downloadURL).toContain('X-Amz-Signature');+ });+ });++ describe('extractKeyFromS3Url', () => {+ it('extracts key from signed URL', async () => {+ const { saveBufferToS3, extractKeyFromS3Url } = await import('../crud');+ const buffer = Buffer.from('extract key test');+ const fileName = `extract-key-${Date.now()}.txt`;++ const signedUrl = await saveBufferToS3({+ userId: TEST_USER_ID,+ buffer,+ fileName,+ basePath: TEST_BASE_PATH,+ });++ const extractedKey = extractKeyFromS3Url(signedUrl);+ expect(extractedKey).toBe(`${TEST_BASE_PATH}/${TEST_USER_ID}/${fileName}`);+ });++ it('returns key as-is when not a URL', async () => {+ const { extractKeyFromS3Url } = await import('../crud');+ const key = `${TEST_BASE_PATH}/${TEST_USER_ID}/file.txt`;+ expect(extractKeyFromS3Url(key)).toBe(key);+ });+ });++ describe('uploadFileToS3', () => {+ it('uploads file and returns filepath with bytes', async () => {+ const { uploadFileToS3 } = await import('../crud');+ const testContent = 'File upload test content';+ const testFilePath = path.join(tempDir, 'upload-test.txt');+ fs.writeFileSync(testFilePath, testContent);++ const mockReq = {+ user: { id: TEST_USER_ID },+ } as ServerRequest;++ const mockFile = {+ path: testFilePath,+ originalname: 'upload-test.txt',+ fieldname: 'file',+ encoding: '7bit',+ mimetype: 'text/plain',+ size: Buffer.byteLength(testContent),+ stream: fs.createReadStream(testFilePath),+ destination: tempDir,+ filename: 'upload-test.txt',+ buffer: Buffer.from(testContent),+ } as Express.Multer.File;++ const fileId = `file-${Date.now()}`;++ const result = await uploadFileToS3({+ req: mockReq,+ file: mockFile,+ file_id: fileId,+ basePath: TEST_BASE_PATH,+ });++ expect(result.filepath).toBeDefined();+ expect(result.filepath).toContain('X-Amz-Signature');+ expect(result.bytes).toBe(Buffer.byteLength(testContent));+ });++ it('throws error when user is not authenticated', async () => {+ const { uploadFileToS3 } = await import('../crud');+ const mockReq = {} as ServerRequest;+ const mockFile = {+ path: '/fake/path.txt',+ originalname: 'test.txt',+ } as Express.Multer.File;++ await expect(+ uploadFileToS3({+ req: mockReq,+ file: mockFile,+ file_id: 'test-id',+ basePath: TEST_BASE_PATH,+ }),+ ).rejects.toThrow('User not authenticated');+ });+ });++ describe('getS3FileStream', () => {+ it('returns readable stream for existing file', async () => {+ const { saveBufferToS3, getS3FileStream } = await import('../crud');+ const testContent = 'Stream test content';+ const buffer = Buffer.from(testContent);+ const fileName = `stream-test-${Date.now()}.txt`;++ const signedUrl = await saveBufferToS3({+ userId: TEST_USER_ID,+ buffer,+ fileName,+ basePath: TEST_BASE_PATH,+ });++ const mockReq = {+ user: { id: TEST_USER_ID },+ } as ServerRequest;++ const stream = await getS3FileStream(mockReq, signedUrl);++ expect(stream).toBeInstanceOf(Readable);++ const chunks: Uint8Array[] = [];+ for await (const chunk of stream) {+ chunks.push(chunk as Uint8Array);+ }+ const downloadedContent = Buffer.concat(chunks).toString();+ expect(downloadedContent).toBe(testContent);+ });+ });++ describe('needsRefresh', () => {+ it('returns false for non-signed URLs', async () => {+ const { needsRefresh } = await import('../crud');+ expect(needsRefresh('https://example.com/file.png', 3600)).toBe(false);+ });++ it('returns true for expired signed URLs', async () => {+ const { saveBufferToS3, needsRefresh } = await import('../crud');+ const buffer = Buffer.from('refresh test');+ const fileName = `refresh-test-${Date.now()}.txt`;++ const signedUrl = await saveBufferToS3({+ userId: TEST_USER_ID,+ buffer,+ fileName,+ basePath: TEST_BASE_PATH,+ });++ const result = needsRefresh(signedUrl, 999999);+ expect(result).toBe(true);+ });++ it('returns false for fresh signed URLs', async () => {+ const { saveBufferToS3, needsRefresh } = await import('../crud');+ const buffer = Buffer.from('fresh test');+ const fileName = `fresh-test-${Date.now()}.txt`;++ const signedUrl = await saveBufferToS3({+ userId: TEST_USER_ID,+ buffer,+ fileName,+ basePath: TEST_BASE_PATH,+ });++ const result = needsRefresh(signedUrl, 60);+ expect(result).toBe(false);+ });+ });++ describe('getNewS3URL', () => {+ it('generates signed URL from existing URL', async () => {+ const { saveBufferToS3, getNewS3URL } = await import('../crud');+ const buffer = Buffer.from('new url test');+ const fileName = `new-url-${Date.now()}.txt`;++ const originalUrl = await saveBufferToS3({+ userId: TEST_USER_ID,+ buffer,+ fileName,+ basePath: TEST_BASE_PATH,+ });++ const newUrl = await getNewS3URL(originalUrl);++ expect(newUrl).toBeDefined();+ expect(newUrl).toContain('X-Amz-Signature');+ expect(newUrl).toContain(fileName);+ });+ });++ describe('refreshS3Url', () => {+ it('returns original URL for non-S3 source', async () => {+ const { refreshS3Url } = await import('../crud');+ const fileObj = {+ filepath: 'https://example.com/file.png',+ source: 'local',+ };++ const result = await refreshS3Url(fileObj, 3600);+ expect(result).toBe(fileObj.filepath);+ });++ it('refreshes URL for S3 source when needed', async () => {+ const { saveBufferToS3, refreshS3Url } = await import('../crud');+ const buffer = Buffer.from('s3 refresh test');+ const fileName = `s3-refresh-${Date.now()}.txt`;++ const originalUrl = await saveBufferToS3({+ userId: TEST_USER_ID,+ buffer,+ fileName,+ basePath: TEST_BASE_PATH,+ });++ const fileObj = {+ filepath: originalUrl,+ source: 's3',+ };++ const newUrl = await refreshS3Url(fileObj, 999999);++ expect(newUrl).toBeDefined();+ expect(newUrl).toContain('X-Amz-Signature');+ });+ });++ describe('S3ImageService', () => {+ it('uploads avatar and returns URL', async () => {+ const { S3ImageService } = await import('../images');++ const mockDeps = {+ resizeImageBuffer: jest.fn().mockImplementation(async (buffer: Buffer) => ({+ buffer,+ width: 100,+ height: 100,+ })),+ updateUser: jest.fn().mockResolvedValue(undefined),+ updateFile: jest.fn().mockResolvedValue(undefined),+ };++ const imageService = new S3ImageService(mockDeps);++ const pngBuffer = MINIMAL_PNG;++ const result = await imageService.processAvatar({+ buffer: pngBuffer,+ userId: TEST_USER_ID,+ manual: 'false',+ basePath: TEST_BASE_PATH,+ });++ expect(result).toBeDefined();+ expect(result).toContain('X-Amz-Signature');+ expect(result).toContain('avatar');+ });++ it('updates user when manual is true', async () => {+ const { S3ImageService } = await import('../images');++ const mockDeps = {+ resizeImageBuffer: jest.fn().mockImplementation(async (buffer: Buffer) => ({+ buffer,+ width: 100,+ height: 100,+ })),+ updateUser: jest.fn().mockResolvedValue(undefined),+ updateFile: jest.fn().mockResolvedValue(undefined),+ };++ const imageService = new S3ImageService(mockDeps);++ const pngBuffer = MINIMAL_PNG;++ await imageService.processAvatar({+ buffer: pngBuffer,+ userId: TEST_USER_ID,+ manual: 'true',+ basePath: TEST_BASE_PATH,+ });++ expect(mockDeps.updateUser).toHaveBeenCalledWith(+ TEST_USER_ID,+ expect.objectContaining({ avatar: expect.any(String) }),+ );+ });++ it('does not update user when agentId is provided', async () => {+ const { S3ImageService } = await import('../images');++ const mockDeps = {+ resizeImageBuffer: jest.fn().mockImplementation(async (buffer: Buffer) => ({+ buffer,+ width: 100,+ height: 100,+ })),+ updateUser: jest.fn().mockResolvedValue(undefined),+ updateFile: jest.fn().mockResolvedValue(undefined),+ };++ const imageService = new S3ImageService(mockDeps);++ const pngBuffer = MINIMAL_PNG;++ await imageService.processAvatar({+ buffer: pngBuffer,+ userId: TEST_USER_ID,+ manual: 'true',+ agentId: 'agent-123',+ basePath: TEST_BASE_PATH,+ });++ expect(mockDeps.updateUser).not.toHaveBeenCalled();+ });++ it('returns tuple with resolved promise and filepath in prepareImageURL', async () => {+ const { S3ImageService } = await import('../images');++ const mockDeps = {+ resizeImageBuffer: jest.fn().mockImplementation(async (buffer: Buffer) => ({+ buffer,+ width: 100,+ height: 100,+ })),+ updateUser: jest.fn().mockResolvedValue(undefined),+ updateFile: jest.fn().mockResolvedValue(undefined),+ };++ const imageService = new S3ImageService(mockDeps);++ const testFile = {+ file_id: 'file-123',+ filepath: 'https://example.com/file.png',+ };++ const result = await imageService.prepareImageURL(testFile);++ expect(Array.isArray(result)).toBe(true);+ expect(result[1]).toBe(testFile.filepath);+ expect(mockDeps.updateFile).toHaveBeenCalledWith({ file_id: 'file-123' });+ });+ });+});diff --git a/packages/api/src/storage/s3/crud.ts b/packages/api/src/storage/s3/crud.tsnew file mode 100644index 0000000000..1143a7ed7f--- /dev/null+++ b/packages/api/src/storage/s3/crud.ts@@ -0,0 +1,460 @@+import fs from 'fs';+import { Readable } from 'stream';+import { logger } from '@librechat/data-schemas';+import { FileSources } from 'librechat-data-provider';+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';+import {+ PutObjectCommand,+ GetObjectCommand,+ HeadObjectCommand,+ DeleteObjectCommand,+} from '@aws-sdk/client-s3';+import type { GetObjectCommandInput } from '@aws-sdk/client-s3';+import type { TFile } from 'librechat-data-provider';+import type { ServerRequest } from '~/types';+import type {+ UploadFileParams,+ SaveBufferParams,+ BatchUpdateFn,+ SaveURLParams,+ GetURLParams,+ UploadResult,+ S3FileRef,+} from '~/storage/types';+import { initializeS3 } from '~/cdn/s3';+import { deleteRagFile } from '~/files';+import { s3Config } from './s3Config';++const {+ AWS_BUCKET_NAME: bucketName,+ AWS_ENDPOINT_URL: endpoint,+ AWS_FORCE_PATH_STYLE: forcePathStyle,+ S3_URL_EXPIRY_SECONDS: s3UrlExpirySeconds,+ S3_REFRESH_EXPIRY_MS: s3RefreshExpiryMs,+ DEFAULT_BASE_PATH: defaultBasePath,+} = s3Config;++export const getS3Key = (basePath: string, userId: string, fileName: string): string => {+ if (basePath.includes('/')) {+ throw new Error(`[getS3Key] basePath must not contain slashes: "${basePath}"`);+ }+ return `${basePath}/${userId}/${fileName}`;+};++export async function getS3URL({+ userId,+ fileName,+ basePath = defaultBasePath,+ customFilename = null,+ contentType = null,+}: GetURLParams): Promise<string> {+ const key = getS3Key(basePath, userId, fileName);+ const params: GetObjectCommandInput = { Bucket: bucketName, Key: key };++ if (customFilename) {+ const safeFilename = customFilename.replace(/["\r\n]/g, '');+ params.ResponseContentDisposition = `attachment; filename="${safeFilename}"`;+ }+ if (contentType) {+ params.ResponseContentType = contentType;+ }++ try {+ const s3 = initializeS3();+ if (!s3) {+ throw new Error('[getS3URL] S3 not initialized');+ }++ return await getSignedUrl(s3, new GetObjectCommand(params), { expiresIn: s3UrlExpirySeconds });+ } catch (error) {+ logger.error('[getS3URL] Error getting signed URL from S3:', (error as Error).message);+ throw error;+ }+}++export async function saveBufferToS3({+ userId,+ buffer,+ fileName,+ basePath = defaultBasePath,+}: SaveBufferParams): Promise<string> {+ const key = getS3Key(basePath, userId, fileName);+ const params = { Bucket: bucketName, Key: key, Body: buffer };++ try {+ const s3 = initializeS3();+ if (!s3) {+ throw new Error('[saveBufferToS3] S3 not initialized');+ }++ await s3.send(new PutObjectCommand(params));+ return await getS3URL({ userId, fileName, basePath });+ } catch (error) {+ logger.error('[saveBufferToS3] Error uploading buffer to S3:', (error as Error).message);+ throw error;+ }+}++export async function saveURLToS3({+ userId,+ URL,+ fileName,+ basePath = defaultBasePath,+}: SaveURLParams): Promise<string> {+ try {+ const response = await fetch(URL);+ if (!response.ok) {+ throw new Error(`Failed to fetch URL: ${response.status} ${response.statusText}`);+ }+ const arrayBuffer = await response.arrayBuffer();+ const buffer = Buffer.from(arrayBuffer);+ return await saveBufferToS3({ userId, buffer, fileName, basePath });+ } catch (error) {+ logger.error('[saveURLToS3] Error uploading file from URL to S3:', (error as Error).message);+ throw error;+ }+}++export function extractKeyFromS3Url(fileUrlOrKey: string): string {+ if (!fileUrlOrKey) {+ throw new Error('Invalid input: URL or key is empty');+ }++ try {+ const url = new URL(fileUrlOrKey);+ const hostname = url.hostname;+ const pathname = url.pathname.substring(1);++ if (endpoint && forcePathStyle) {+ const endpointUrl = new URL(endpoint);+ const startPos =+ endpointUrl.pathname.length ++ (endpointUrl.pathname.endsWith('/') ? 0 : 1) ++ bucketName.length ++ 1;+ const key = url.pathname.substring(startPos);+ if (!key) {+ logger.warn(+ `[extractKeyFromS3Url] Extracted key is empty for endpoint path-style URL: ${fileUrlOrKey}`,+ );+ } else {+ logger.debug(`[extractKeyFromS3Url] fileUrlOrKey: ${fileUrlOrKey}, Extracted key: ${key}`);+ }+ return key;+ }++ if (+ hostname === 's3.amazonaws.com' ||+ hostname.match(/^s3[-.][a-z0-9-]+\.amazonaws\.com$/) ||+ (bucketName && pathname.startsWith(`${bucketName}/`))+ ) {+ const firstSlashIndex = pathname.indexOf('/');+ if (firstSlashIndex > 0) {+ const key = pathname.substring(firstSlashIndex + 1);+ if (key === '') {+ logger.warn(+ `[extractKeyFromS3Url] Extracted key is empty after removing bucket name from URL: ${fileUrlOrKey}`,+ );+ } else {+ logger.debug(+ `[extractKeyFromS3Url] fileUrlOrKey: ${fileUrlOrKey}, Extracted key: ${key}`,+ );+ }+ return key;+ }+ logger.warn(+ `[extractKeyFromS3Url] Unable to extract key from path-style URL: ${fileUrlOrKey}`,+ );+ return '';+ }++ logger.debug(`[extractKeyFromS3Url] fileUrlOrKey: ${fileUrlOrKey}, Extracted key: ${pathname}`);+ return pathname;+ } catch (error) {+ if (fileUrlOrKey.startsWith('http://') || fileUrlOrKey.startsWith('https://')) {+ logger.error(+ `[extractKeyFromS3Url] Error parsing URL: ${fileUrlOrKey}, Error: ${(error as Error).message}`,+ );+ } else {+ logger.debug(`[extractKeyFromS3Url] Non-URL input, using fallback: ${fileUrlOrKey}`);+ }++ const parts = fileUrlOrKey.split('/');+ if (parts.length >= 3 && !fileUrlOrKey.startsWith('http') && !fileUrlOrKey.startsWith('/')) {+ return fileUrlOrKey;+ }++ const key = fileUrlOrKey.startsWith('/') ? fileUrlOrKey.substring(1) : fileUrlOrKey;+ logger.debug(+ `[extractKeyFromS3Url] FALLBACK. fileUrlOrKey: ${fileUrlOrKey}, Extracted key: ${key}`,+ );+ return key;+ }+}++export async function deleteFileFromS3(req: ServerRequest, file: TFile): Promise<void> {+ if (!req.user) {+ throw new Error('[deleteFileFromS3] User not authenticated');+ }++ const userId = req.user.id;+ const key = extractKeyFromS3Url(file.filepath);++ const keyParts = key.split('/');+ if (keyParts.length < 2 || keyParts[1] !== userId) {+ const message = `[deleteFileFromS3] User ID mismatch: ${userId} vs ${key}`;+ logger.error(message);+ throw new Error(message);+ }++ const s3 = initializeS3();+ if (!s3) {+ throw new Error('[deleteFileFromS3] S3 not initialized');+ }++ const params = { Bucket: bucketName, Key: key };++ try {+ try {+ const headCommand = new HeadObjectCommand(params);+ await s3.send(headCommand);+ logger.debug('[deleteFileFromS3] File exists, proceeding with deletion');+ } catch (headErr) {+ if ((headErr as { name?: string }).name === 'NotFound') {+ logger.warn(`[deleteFileFromS3] File does not exist: ${key}`);+ await deleteRagFile({ userId, file });+ return;+ }+ throw headErr;+ }++ await s3.send(new DeleteObjectCommand(params));+ await deleteRagFile({ userId, file });+ logger.debug('[deleteFileFromS3] S3 File deletion completed');+ } catch (error) {+ logger.error(`[deleteFileFromS3] Error deleting file from S3: ${(error as Error).message}`);+ logger.error((error as Error).stack);++ if ((error as { name?: string }).name === 'NoSuchKey') {+ return;+ }+ throw error;+ }+}++export async function uploadFileToS3({+ req,+ file,+ file_id,+ basePath = defaultBasePath,+}: UploadFileParams): Promise<UploadResult> {+ if (!req.user) {+ throw new Error('[uploadFileToS3] User not authenticated');+ }++ try {+ const inputFilePath = file.path;+ const userId = req.user.id;+ const fileName = `${file_id}__${file.originalname}`;+ const key = getS3Key(basePath, userId, fileName);++ const stats = await fs.promises.stat(inputFilePath);+ const bytes = stats.size;+ const fileStream = fs.createReadStream(inputFilePath);++ const s3 = initializeS3();+ if (!s3) {+ throw new Error('[uploadFileToS3] S3 not initialized');+ }++ const uploadParams = {+ Bucket: bucketName,+ Key: key,+ Body: fileStream,+ };++ await s3.send(new PutObjectCommand(uploadParams));+ const fileURL = await getS3URL({ userId, fileName, basePath });+ // NOTE: temp file is intentionally NOT deleted on the success path.+ // The caller (processAgentFileUpload) reads file.path after this returns+ // to stream the file to the RAG vector embedding service (POST /embed).+ // Temp file lifecycle on success is the caller's responsibility.+ return { filepath: fileURL, bytes };+ } catch (error) {+ logger.error('[uploadFileToS3] Error streaming file to S3:', error);+ if (file?.path) {+ await fs.promises+ .unlink(file.path)+ .catch((e: unknown) =>+ logger.error('[uploadFileToS3] Failed to delete temp file:', (e as Error).message),+ );+ }+ throw error;+ }+}++export async function getS3FileStream(_req: ServerRequest, filePath: string): Promise<Readable> {+ try {+ const Key = extractKeyFromS3Url(filePath);+ const params = { Bucket: bucketName, Key };++ const s3 = initializeS3();+ if (!s3) {+ throw new Error('[getS3FileStream] S3 not initialized');+ }++ const data = await s3.send(new GetObjectCommand(params));+ if (!data.Body) {+ throw new Error(`[getS3FileStream] S3 response body is empty for key: ${Key}`);+ }+ return data.Body as Readable;+ } catch (error) {+ logger.error('[getS3FileStream] Error retrieving S3 file stream:', error);+ throw error;+ }+}++export function needsRefresh(signedUrl: string, bufferSeconds: number): boolean {+ try {+ const url = new URL(signedUrl);++ if (!url.searchParams.has('X-Amz-Signature')) {+ return false;+ }++ const expiresParam = url.searchParams.get('X-Amz-Expires');+ const dateParam = url.searchParams.get('X-Amz-Date');++ if (!expiresParam || !dateParam) {+ return true;+ }++ const year = dateParam.substring(0, 4);+ const month = dateParam.substring(4, 6);+ const day = dateParam.substring(6, 8);+ const hour = dateParam.substring(9, 11);+ const minute = dateParam.substring(11, 13);+ const second = dateParam.substring(13, 15);++ const dateObj = new Date(`${year}-${month}-${day}T${hour}:${minute}:${second}Z`);+ const now = new Date();++ if (s3RefreshExpiryMs !== null) {+ const urlAge = now.getTime() - dateObj.getTime();+ return urlAge >= s3RefreshExpiryMs;+ }++ const expiresAtDate = new Date(dateObj.getTime() + parseInt(expiresParam) * 1000);+ const bufferTime = new Date(now.getTime() + bufferSeconds * 1000);+ return expiresAtDate <= bufferTime;+ } catch (error) {+ logger.error('Error checking URL expiration:', error);+ return true;+ }+}++export async function getNewS3URL(currentURL: string): Promise<string | undefined> {+ try {+ const s3Key = extractKeyFromS3Url(currentURL);+ if (!s3Key) {+ return;+ }++ const keyParts = s3Key.split('/');+ if (keyParts.length < 3) {+ return;+ }++ const basePath = keyParts[0];+ const userId = keyParts[1];+ const fileName = keyParts.slice(2).join('/');++ return getS3URL({ userId, fileName, basePath });+ } catch (error) {+ logger.error('Error getting new S3 URL:', error);+ }+}++export async function refreshS3FileUrls(+ files: TFile[] | null | undefined,+ batchUpdateFiles: BatchUpdateFn,+ bufferSeconds = 3600,+): Promise<TFile[]> {+ if (!files || !Array.isArray(files) || files.length === 0) {+ return [];+ }++ const filesToUpdate: Array<{ file_id: string; filepath: string }> = [];+ const updatedFiles = [...files];++ for (let i = 0; i < updatedFiles.length; i++) {+ const file = updatedFiles[i];+ if (!file?.file_id) {+ continue;+ }+ if (file.source !== FileSources.s3) {+ continue;+ }+ if (!file.filepath) {+ continue;+ }+ if (!needsRefresh(file.filepath, bufferSeconds)) {+ continue;+ }++ try {+ const newURL = await getNewS3URL(file.filepath);+ if (!newURL) {+ continue;+ }+ filesToUpdate.push({+ file_id: file.file_id,+ filepath: newURL,+ });+ updatedFiles[i] = { ...file, filepath: newURL };+ } catch (error) {+ logger.error(`Error refreshing S3 URL for file ${file.file_id}:`, error);+ }+ }++ if (filesToUpdate.length > 0) {+ await batchUpdateFiles(filesToUpdate);+ }++ return updatedFiles;+}++export async function refreshS3Url(fileObj: S3FileRef, bufferSeconds = 3600): Promise<string> {+ if (!fileObj || fileObj.source !== FileSources.s3 || !fileObj.filepath) {+ return fileObj?.filepath || '';+ }++ if (!needsRefresh(fileObj.filepath, bufferSeconds)) {+ return fileObj.filepath;+ }++ try {+ const s3Key = extractKeyFromS3Url(fileObj.filepath);+ if (!s3Key) {+ logger.warn(`Unable to extract S3 key from URL: ${fileObj.filepath}`);+ return fileObj.filepath;+ }++ const keyParts = s3Key.split('/');+ if (keyParts.length < 3) {+ logger.warn(`Invalid S3 key format: ${s3Key}`);+ return fileObj.filepath;+ }++ const basePath = keyParts[0];+ const userId = keyParts[1];+ const fileName = keyParts.slice(2).join('/');++ const newUrl = await getS3URL({ userId, fileName, basePath });+ logger.debug(`Refreshed S3 URL for key: ${s3Key}`);+ return newUrl;+ } catch (error) {+ logger.error(`Error refreshing S3 URL: ${(error as Error).message}`);+ return fileObj.filepath;+ }+}diff --git a/packages/api/src/storage/s3/images.ts b/packages/api/src/storage/s3/images.tsnew file mode 100644index 0000000000..b9d7322359--- /dev/null+++ b/packages/api/src/storage/s3/images.ts@@ -0,0 +1,141 @@+import fs from 'fs';+import path from 'path';+import sharp from 'sharp';+import { logger } from '@librechat/data-schemas';+import type { IUser } from '@librechat/data-schemas';+import type { TFile } from 'librechat-data-provider';+import type { FormatEnum } from 'sharp';+import type { UploadImageParams, ImageUploadResult, ProcessAvatarParams } from '~/storage/types';+import { saveBufferToS3 } from './crud';+import { s3Config } from './s3Config';++const { DEFAULT_BASE_PATH: defaultBasePath } = s3Config;++export interface S3ImageServiceDeps {+ resizeImageBuffer: (+ buffer: Buffer,+ resolution: string,+ endpoint: string,+ ) => Promise<{ buffer: Buffer; width: number; height: number }>;+ updateUser: (userId: string, update: { avatar: string }) => Promise<IUser | null>;+ updateFile: (params: { file_id: string }) => Promise<TFile>;+}++export class S3ImageService {+ private deps: S3ImageServiceDeps;++ constructor(deps: S3ImageServiceDeps) {+ this.deps = deps;+ }++ async uploadImageToS3({+ req,+ file,+ file_id,+ endpoint,+ resolution = 'high',+ basePath = defaultBasePath,+ }: UploadImageParams): Promise<ImageUploadResult> {+ const inputFilePath = file.path;+ try {+ if (!req.user) {+ throw new Error('[S3ImageService.uploadImageToS3] User not authenticated');+ }++ const appConfig = req.config;+ const inputBuffer = await fs.promises.readFile(inputFilePath);++ const {+ buffer: resizedBuffer,+ width,+ height,+ } = await this.deps.resizeImageBuffer(inputBuffer, resolution, endpoint);++ const extension = path.extname(inputFilePath);+ const userId = req.user.id;++ let processedBuffer: Buffer;+ let fileName = `${file_id}__${path.basename(inputFilePath)}`;+ const targetExtension = `.${appConfig?.imageOutputType ?? 'webp'}`;++ if (extension.toLowerCase() === targetExtension) {+ processedBuffer = resizedBuffer;+ } else {+ const outputFormat = (appConfig?.imageOutputType ?? 'webp') as keyof FormatEnum;+ processedBuffer = await sharp(resizedBuffer).toFormat(outputFormat).toBuffer();+ fileName = fileName.replace(new RegExp(path.extname(fileName) + '$'), targetExtension);+ if (!path.extname(fileName)) {+ fileName += targetExtension;+ }+ }++ const downloadURL = await saveBufferToS3({+ userId,+ buffer: processedBuffer,+ fileName,+ basePath,+ });+ const bytes = processedBuffer.length;+ return { filepath: downloadURL, bytes, width, height };+ } catch (error) {+ logger.error(+ '[S3ImageService.uploadImageToS3] Error uploading image to S3:',+ (error as Error).message,+ );+ throw error;+ } finally {+ await fs.promises+ .unlink(inputFilePath)+ .catch((e: unknown) =>+ logger.error(+ '[S3ImageService.uploadImageToS3] Failed to delete temp file:',+ (e as Error).message,+ ),+ );+ }+ }++ async prepareImageURL(file: { file_id: string; filepath: string }): Promise<[TFile, string]> {+ try {+ return await Promise.all([this.deps.updateFile({ file_id: file.file_id }), file.filepath]);+ } catch (error) {+ logger.error(+ '[S3ImageService.prepareImageURL] Error preparing image URL:',+ (error as Error).message,+ );+ throw error;+ }+ }++ async processAvatar({+ buffer,+ userId,+ manual,+ agentId,+ basePath = defaultBasePath,+ }: ProcessAvatarParams): Promise<string> {+ try {+ const metadata = await sharp(buffer).metadata();+ const extension = metadata.format ?? 'png';+ const timestamp = new Date().getTime();++ const fileName = agentId+ ? `agent-${agentId}-avatar-${timestamp}.${extension}`+ : `avatar-${timestamp}.${extension}`;++ const downloadURL = await saveBufferToS3({ userId, buffer, fileName, basePath });++ if (manual === 'true' && !agentId) {+ await this.deps.updateUser(userId, { avatar: downloadURL });+ }++ return downloadURL;+ } catch (error) {+ logger.error(+ '[S3ImageService.processAvatar] Error processing S3 avatar:',+ (error as Error).message,+ );+ throw error;+ }+ }+}diff --git a/packages/api/src/storage/s3/index.ts b/packages/api/src/storage/s3/index.tsnew file mode 100644index 0000000000..e700610bba--- /dev/null+++ b/packages/api/src/storage/s3/index.ts@@ -0,0 +1,2 @@+export * from './crud';+export * from './images';diff --git a/packages/api/src/storage/s3/s3Config.ts b/packages/api/src/storage/s3/s3Config.tsnew file mode 100644index 0000000000..766c0cf66e--- /dev/null+++ b/packages/api/src/storage/s3/s3Config.ts@@ -0,0 +1,57 @@+import { logger } from '@librechat/data-schemas';+import { isEnabled } from '~/utils/common';++const MAX_EXPIRY_SECONDS = 7 * 24 * 60 * 60; // 7 days+const DEFAULT_EXPIRY_SECONDS = 2 * 60; // 2 minutes+const DEFAULT_BASE_PATH = 'images';++const parseUrlExpiry = (): number => {+ if (process.env.S3_URL_EXPIRY_SECONDS === undefined) {+ return DEFAULT_EXPIRY_SECONDS;+ }++ const parsed = parseInt(process.env.S3_URL_EXPIRY_SECONDS, 10);+ if (isNaN(parsed) || parsed <= 0) {+ logger.warn(+ `[S3] Invalid S3_URL_EXPIRY_SECONDS value: "${process.env.S3_URL_EXPIRY_SECONDS}". Using ${DEFAULT_EXPIRY_SECONDS}s expiry.`,+ );+ return DEFAULT_EXPIRY_SECONDS;+ }++ return Math.min(parsed, MAX_EXPIRY_SECONDS);+};++const parseRefreshExpiry = (): number | null => {+ if (!process.env.S3_REFRESH_EXPIRY_MS) {+ return null;+ }++ const parsed = parseInt(process.env.S3_REFRESH_EXPIRY_MS, 10);+ if (isNaN(parsed) || parsed <= 0) {+ logger.warn(+ `[S3] Invalid S3_REFRESH_EXPIRY_MS value: "${process.env.S3_REFRESH_EXPIRY_MS}". Using default refresh logic.`,+ );+ return null;+ }++ logger.info(`[S3] Using custom refresh expiry time: ${parsed}ms`);+ return parsed;+};++// Internal module config — not part of the public @librechat/api surface+export const s3Config = {+ /** AWS region for S3 */+ AWS_REGION: process.env.AWS_REGION ?? '',+ /** S3 bucket name */+ AWS_BUCKET_NAME: process.env.AWS_BUCKET_NAME ?? '',+ /** Custom endpoint URL (for MinIO, R2, etc.) */+ AWS_ENDPOINT_URL: process.env.AWS_ENDPOINT_URL,+ /** Use path-style URLs instead of virtual-hosted-style */+ AWS_FORCE_PATH_STYLE: isEnabled(process.env.AWS_FORCE_PATH_STYLE),+ /** Presigned URL expiry in seconds */+ S3_URL_EXPIRY_SECONDS: parseUrlExpiry(),+ /** Custom refresh expiry in milliseconds (null = use default buffer logic) */+ S3_REFRESH_EXPIRY_MS: parseRefreshExpiry(),+ /** Default base path for file storage */+ DEFAULT_BASE_PATH,+};diff --git a/packages/api/src/storage/types.ts b/packages/api/src/storage/types.tsnew file mode 100644index 0000000000..314719f38a--- /dev/null+++ b/packages/api/src/storage/types.ts@@ -0,0 +1,60 @@+import type { ServerRequest } from '~/types';++export interface SaveBufferParams {+ userId: string;+ buffer: Buffer;+ fileName: string;+ basePath?: string;+}++export interface GetURLParams {+ userId: string;+ fileName: string;+ basePath?: string;+ customFilename?: string | null;+ contentType?: string | null;+}++export interface SaveURLParams {+ userId: string;+ URL: string;+ fileName: string;+ basePath?: string;+}++export interface UploadFileParams {+ req: ServerRequest;+ file: Express.Multer.File;+ file_id: string;+ basePath?: string;+}++export interface UploadImageParams extends UploadFileParams {+ endpoint: string;+ resolution?: string;+}++export interface UploadResult {+ filepath: string;+ bytes: number;+}++export interface ImageUploadResult extends UploadResult {+ width: number;+ height: number;+}++export interface ProcessAvatarParams {+ buffer: Buffer;+ userId: string;+ manual: string;+ agentId?: string;+ basePath?: string;+}++export interface S3FileRef {+ filepath: string;+ source: string;+}++export type BatchUpdateFn = (files: Array<{ file_id: string; filepath: string }>) => Promise<void>;