Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🌳 fix: Preserve Primary Files Until Secondary Cleanup Succeeds (#17228)

lia-by-librechat[bot]a51c45bbf32a
diff --git a/api/server/controllers/__tests__/deleteUser.spec.js b/api/server/controllers/__tests__/deleteUser.spec.js
index 13dafb794b..f043b2886a 100644
--- a/api/server/controllers/__tests__/deleteUser.spec.js
+++ b/api/server/controllers/__tests__/deleteUser.spec.js
@@ -213,7 +213,10 @@ function stubDeletionMocks() {
mockDeleteAllSharedLinks.mockResolvedValue();
mockDeleteAllSharedLinksWithCleanup.mockResolvedValue({ deletedCount: 0 });
mockGetFiles.mockResolvedValue([]);
- mockProcessDeleteRequest.mockResolvedValue({ deletedFileIds: [], failedFileIds: [] });
+ mockProcessDeleteRequest.mockImplementation(async ({ files }) => ({
+ deletedFileIds: files.map((file) => file.file_id),
+ failedFileIds: [],
+ }));
mockDeleteFiles.mockResolvedValue();
mockDeleteToolCalls.mockResolvedValue();
mockDeleteUserAgents.mockResolvedValue();
@@ -265,7 +268,7 @@ describe('deleteUserController - 2FA enforcement', () => {
const req = { user: { id: USER_ID, _id: USER_ID, email: '[email protected]' } };
const res = createRes();
mockGetUserById.mockResolvedValue({ _id: USER_ID, twoFactorEnabled: false });
- mockGetFiles.mockResolvedValue(files);
+ mockGetFiles.mockResolvedValueOnce(files);
await deleteUserController(req, res);
@@ -276,6 +279,27 @@ describe('deleteUserController - 2FA enforcement', () => {
expect(fileRequest.user).toBe(req.user);
});
+ it.each(['reported', 'thrown'])('retains the account on %s failure', async (failure) => {
+ const req = { user: { id: USER_ID, _id: USER_ID }, body: {} };
+ const res = createRes();
+ mockGetUserById.mockResolvedValue({ _id: USER_ID, twoFactorEnabled: false });
+ mockGetFiles.mockResolvedValue([{ file_id: 'file-1', embedded: true }]);
+ if (failure === 'reported') {
+ mockProcessDeleteRequest.mockResolvedValueOnce({
+ deletedFileIds: [],
+ failedFileIds: ['file-1'],
+ });
+ } else {
+ mockProcessDeleteRequest.mockRejectedValueOnce(new Error('provider down'));
+ }
+ await deleteUserController(req, res);
+ expect(res.status).toHaveBeenCalledWith(500);
+ expect(mockDeleteFiles).not.toHaveBeenCalled();
+ expect(mockDeleteUserKey).not.toHaveBeenCalled();
+ expect(mockDeleteUserById).not.toHaveBeenCalled();
+ expect(mockCancelAgentTriggerUserDeletion).toHaveBeenCalled();
+ });
+
it('proceeds with deletion when 2FA is not enabled', async () => {
const req = { user: { id: USER_ID, _id: USER_ID, email: '[email protected]' }, body: {} };
const res = createRes();
diff --git a/api/server/controllers/agents/__tests__/openai.spec.js b/api/server/controllers/agents/__tests__/openai.spec.js
index f9ea444636..6695e9d04b 100644
--- a/api/server/controllers/agents/__tests__/openai.spec.js
+++ b/api/server/controllers/agents/__tests__/openai.spec.js
@@ -175,6 +175,8 @@ jest.mock('@librechat/agents', () => ({
}));
jest.mock('@librechat/api', () => ({
+ createFileProvisionRecovery: () =>
+ jest.requireActual('@librechat/api').createFileProvisionRecovery(),
getAgentErrorMetadata: (...args) =>
jest.requireActual('@librechat/api').getAgentErrorMetadata(...args),
isTemporaryTurn: (...args) => jest.requireActual('@librechat/api').isTemporaryTurn(...args),
diff --git a/api/server/controllers/agents/__tests__/responses.unit.spec.js b/api/server/controllers/agents/__tests__/responses.unit.spec.js
index 237062a6e6..5d38d8c6c6 100644
--- a/api/server/controllers/agents/__tests__/responses.unit.spec.js
+++ b/api/server/controllers/agents/__tests__/responses.unit.spec.js
@@ -198,6 +198,8 @@ jest.mock('@librechat/agents', () => ({
}));
jest.mock('@librechat/api', () => ({
+ createFileProvisionRecovery: () =>
+ jest.requireActual('@librechat/api').createFileProvisionRecovery(),
getAgentErrorMetadata: (...args) =>
jest.requireActual('@librechat/api').getAgentErrorMetadata(...args),
getConversationWriteContext: (...args) =>
diff --git a/api/server/services/Endpoints/agents/initialize.js b/api/server/services/Endpoints/agents/initialize.js
index f7be7411f1..f65874a490 100644
--- a/api/server/services/Endpoints/agents/initialize.js
+++ b/api/server/services/Endpoints/agents/initialize.js
@@ -117,6 +117,7 @@ const {
} = require('./backgroundCompletion');
const { logViolation } = require('~/cache');
const db = require('~/models');
+const { getFileStorage } = require('~/server/services/Files/operations');
const { getAppConfig } = require('~/server/services/Config');
const { getStrategyFunctions } = require('~/server/services/Files/strategies');
const { encodeAndFormat } = require('~/server/services/Files/images/encode');
@@ -1992,6 +1993,7 @@ const initializeClientWithProvider = async ({
filterFiles: filterFilesByAgentAccess,
listPublications: db.listRunArtifacts,
provisioning: {
+ fileStorage: getFileStorage(),
provisionToCodeEnv,
provisionToVectorDB,
updateFile: db.updateFile,
diff --git a/api/server/services/Files/Azure/crud.js b/api/server/services/Files/Azure/crud.js
index ee2cb8e40d..1f3f2c958a 100644
--- a/api/server/services/Files/Azure/crud.js
+++ b/api/server/services/Files/Azure/crud.js
@@ -5,6 +5,7 @@ const fetch = require('node-fetch');
const { logger } = require('@librechat/data-schemas');
const {
deleteRagFile,
+ dispatchFileDeletion,
createOwnedBlobFileDeleter,
extractAzureBlobPath,
assertRemoteFileURL,
@@ -122,13 +123,13 @@ async function getAzureURL({ fileName, basePath = defaultBasePath, userId, conta
* @param {ServerRequest} params.req - The Express request object.
* @param {MongoFile} params.file - The file object.
*/
-const deleteFileFromAzure = (req, file) =>
+const deleteFileFromAzure = (req, file, _client, onDelete) =>
createOwnedBlobFileDeleter({
extractPath: (filepath) => extractAzureBlobPath(filepath, AZURE_CONTAINER_NAME),
deleteBlob: async (blobPath) => {
const containerClient = await getAzureContainerClient(AZURE_CONTAINER_NAME);
const blockBlobClient = containerClient.getBlockBlobClient(blobPath);
- await blockBlobClient.delete();
+ await dispatchFileDeletion(onDelete, () => blockBlobClient.delete());
logger.debug('[deleteFileFromAzure] Blob deleted successfully from Azure Blob Storage');
},
isMissing: (error) => error?.statusCode === 404,
diff --git a/api/server/services/Files/Azure/crud.spec.js b/api/server/services/Files/Azure/crud.spec.js
index bafbb98430..ba6fdbac09 100644
--- a/api/server/services/Files/Azure/crud.spec.js
+++ b/api/server/services/Files/Azure/crud.spec.js
@@ -1,5 +1,6 @@
const mockDownload = jest.fn();
-const mockGetBlockBlobClient = jest.fn(() => ({ download: mockDownload }));
+const mockDelete = jest.fn();
+const mockGetBlockBlobClient = jest.fn(() => ({ download: mockDownload, delete: mockDelete }));
const mockGetAzureContainerClient = jest.fn(async () => ({
url: 'https://account.blob.core.windows.net/files',
getBlockBlobClient: mockGetBlockBlobClient,
@@ -7,10 +8,13 @@ const mockGetAzureContainerClient = jest.fn(async () => ({
const mockGetSafeErrorMetadata = jest.fn(() => ({ type: 'Error', status: 403 }));
jest.mock('@librechat/data-schemas', () => ({
- logger: { error: jest.fn() },
+ logger: { error: jest.fn(), debug: jest.fn() },
}));
jest.mock('@librechat/api', () => ({
+ ...jest.requireActual('../../../../../packages/api/src/storage/ownedBlobDelete'),
+ dispatchFileDeletion: jest.requireActual('../../../../../packages/api/src/files/dispatch')
+ .dispatchFileDeletion,
deleteRagFile: jest.fn(),
assertRemoteFileURL: jest.fn((url) => url),
getSafeErrorMetadata: (...args) => mockGetSafeErrorMetadata(...args),
@@ -21,7 +25,8 @@ jest.mock('@librechat/api', () => ({
}));
const { logger } = require('@librechat/data-schemas');
-const { getAzureFileStream } = require('./crud');
+const { getAzureFileStream, deleteFileFromAzure } = require('./crud');
+const { deleteRagFile } = require('@librechat/api');
describe('getAzureFileStream', () => {
it('downloads private blobs through the authenticated Azure client', async () => {
@@ -76,3 +81,51 @@ describe('getAzureFileStream', () => {
expect(JSON.stringify(logger.error.mock.calls)).not.toContain(signedUrl);
});
});
+
+describe('deleteFileFromAzure', () => {
+ const req = { user: { id: 'owner' } };
+ const file = { filepath: 'https://account.blob.core.windows.net/files/uploads/owner/file.txt' };
+
+ beforeEach(() => {
+ mockDelete.mockReset().mockResolvedValue(undefined);
+ deleteRagFile.mockReset().mockResolvedValue(undefined);
+ });
+
+ it('does not mark a failed client preflight', async () => {
+ const onDelete = jest.fn();
+ const error = Object.assign(new Error('client timed out'), { code: 'ETIMEDOUT' });
+ mockGetAzureContainerClient.mockRejectedValueOnce(error);
+ await expect(deleteFileFromAzure(req, file, undefined, onDelete)).rejects.toBe(error);
+ expect(onDelete).not.toHaveBeenCalled();
+ expect(mockDelete).not.toHaveBeenCalled();
+ });
+
+ it('does not mark invalid ownership before dispatch', async () => {
+ const onDelete = jest.fn();
+ await expect(
+ deleteFileFromAzure({ user: { id: 'other' } }, file, undefined, onDelete),
+ ).rejects.toThrow('Invalid file path');
+ expect(onDelete).not.toHaveBeenCalled();
+ expect(mockDelete).not.toHaveBeenCalled();
+ });
+
+ it('marks a lost delete response without confirming deletion', async () => {
+ const onDelete = jest.fn();
+ const error = Object.assign(new Error('response lost'), { code: 'ETIMEDOUT' });
+ mockDelete.mockRejectedValueOnce(error);
+ await expect(deleteFileFromAzure(req, file, undefined, onDelete)).rejects.toBe(error);
+ expect(onDelete.mock.calls).toEqual([[]]);
+ expect(deleteRagFile).not.toHaveBeenCalled();
+ });
+
+ it('confirms deletion before a later RAG denial', async () => {
+ const onDelete = jest.fn();
+ const error = Object.assign(new Error('RAG denied'), { statusCode: 403 });
+ deleteRagFile.mockImplementationOnce(async () => {
+ expect(onDelete.mock.calls).toEqual([[], ['confirmed']]);
+ throw error;
+ });
+ await expect(deleteFileFromAzure(req, file, undefined, onDelete)).rejects.toBe(error);
+ expect(onDelete.mock.calls).toEqual([[], ['confirmed']]);
+ });
+});
diff --git a/api/server/services/Files/Code/__tests__/process-traversal.spec.js b/api/server/services/Files/Code/__tests__/process-traversal.spec.js
index b7ff244611..393736b79a 100644
--- a/api/server/services/Files/Code/__tests__/process-traversal.spec.js
+++ b/api/server/services/Files/Code/__tests__/process-traversal.spec.js
@@ -22,6 +22,7 @@ jest.mock('@librechat/api', () => {
const https = require('https');
return {
processCodeOutput: jest.requireActual('@librechat/api').processCodeOutput,
+ createFileProvisionRecovery: jest.requireActual('@librechat/api').createFileProvisionRecovery,
createSandboxTextReader: jest.requireActual('@librechat/api').createSandboxTextReader,
resolveDownloadPath: (file) => file.storageKey || file.filepath,
logAxiosError: jest.fn(),
@@ -82,6 +83,12 @@ jest.mock('~/models', () => ({
updateFile: jest.fn(),
claimCodeFile: jest.fn().mockResolvedValue({ file_id: 'mock-uuid', usage: 0 }),
commitCodeFile: jest.fn().mockResolvedValue(true),
+ acquireFileProvision: jest.fn().mockResolvedValue(true),
+ releaseFileProvision: jest.fn().mockResolvedValue(true),
+ commitFileProvision: jest.fn(async ({ output, releaseOnly }) => {
+ if (releaseOnly) return true;
+ return jest.requireMock('~/models').commitCodeFile(output.file, output.sourceDispatchedAt);
+ }),
}));
const mockSaveBuffer = jest.fn().mockResolvedValue('/uploads/user123/mock-uuid__output.csv');
diff --git a/api/server/services/Files/Code/crud.js b/api/server/services/Files/Code/crud.js
index e6e625f048..d242dfe5f7 100644
--- a/api/server/services/Files/Code/crud.js
+++ b/api/server/services/Files/Code/crud.js
@@ -1,9 +1,11 @@
const FormData = require('form-data');
const { logger } = require('@librechat/data-schemas');
const { getCodeBaseURL } = require('@librechat/agents');
-const { EModelEndpoint, getCodeEnvRefs } = require('librechat-data-provider');
+const { clearFileCodeEnvRefs } = require('~/models');
const {
logAxiosError,
+ createCodeFileDeleter,
+ dispatchFileStorageOperation,
wrapCodeApiUploadError,
appendCodeEnvFile,
createAxiosInstance,
@@ -72,106 +74,18 @@ async function getCodeOutputDownloadStream(fileIdentifier, identity, req, route
}
}
-/**
- * A deployment that no longer serves a ref's profile rejects every retry the same
- * way, so failing the delete would leave a record the user can never remove and an
- * agent that aborts every code turn on it.
- */
-const isProfileMismatch = (error) =>
- error.response?.status === 409 && error.response?.data?.error === 'execution_profile_mismatch';
-
-/**
- * Deletes a file from the Code Environment server.
- *
- * @param {ServerRequest} req - Current authenticated request, used to mint Code API auth.
- * @param {MongoFile} file - File metadata containing `metadata.codeEnvRef`.
- * @returns {Promise<void>}
- */
-async function deleteCodeEnvFile(req, file) {
- const refs = getCodeEnvRefs(file?.metadata);
- if (refs.length === 0) {
- return;
- }
-
- for (const [executionRouteKey, ref] of refs) {
- const executionProfile = ref.executionProfile ?? 'default';
- const environments =
- req.config?.endpoints?.[EModelEndpoint.agents]?.statefulCodeSessions?.environments;
- const configuredEnvironment = environments?.find(
- (environment) =>
- createCodeExecutionRouteKey(executionProfile, environment) === executionRouteKey,
- );
- if (
- executionProfile === 'stateful' &&
- executionRouteKey !== executionProfile &&
- !configuredEnvironment
- ) {
- logger.warn(
- `[deleteCodeEnvFile] Skipping remote cleanup for unmapped historical route ${executionRouteKey}`,
- );
- continue;
- }
- let baseURL;
- try {
- baseURL = getCodeExecutionBaseUrl(executionProfile, configuredEnvironment);
- } catch (error) {
- if (
- executionProfile === 'stateful' &&
- executionRouteKey === executionProfile &&
- !configuredEnvironment
- ) {
- logger.warn(
- '[deleteCodeEnvFile] Skipping remote cleanup for retired legacy stateful route',
- );
- continue;
- }
- throw error;
- }
- const query = buildCodeEnvDownloadQuery({
- kind: ref.kind,
- id: ref.id,
- ...(ref.kind === 'skill' ? { version: ref.version } : {}),
- });
- const bridgeWorkerId =
- configuredEnvironment?.workerId ?? configuredEnvironment?.pairing?.workerId;
- const authHeaders = await getCodeApiAuthHeaders(req, bridgeWorkerId);
- /* codeapi has mounted DELETE at `/files/:session_id/:fileId` since its
- * first release. The file-server's own `/sessions/:id/objects/:fileId`
- * only gained DELETE in LibreChat-AI/code-interpreter#85, so trying it
- * first cost a guaranteed 404 against every older deployment. */
- try {
- await axios({
- method: 'delete',
- url: `${baseURL}/files/${ref.storage_session_id}/${ref.file_id}${query}`,
- headers: {
- 'User-Agent': 'LibreChat/1.0',
- ...authHeaders,
- ...codeExecutionHeaders({
- executionProfile,
- bridgeWorkerId,
- }),
- },
- httpAgent: codeServerHttpAgent,
- httpsAgent: codeServerHttpsAgent,
- timeout: 15000,
- });
- } catch (error) {
- const profileMismatch = isProfileMismatch(error);
- if (error.response?.status !== 404 && !profileMismatch) {
- throw error;
- }
- /* Already gone. Logged rather than swallowed: a 404 from a
- * misconfigured base URL is indistinguishable from one for an absent
- * object, and this branch drops the file's record either way. */
- logAxiosError({
- error,
- message: profileMismatch
- ? `Code environment no longer serves the ${executionProfile} profile; dropping the unreachable object: ${error.message}`
- : `Code environment object already absent: ${error.message}`,
- });
- }
- }
-}
+const deleteCodeEnvFile = createCodeFileDeleter({
+ httpClient: axios,
+ getAuthHeaders: getCodeApiAuthHeaders,
+ getBaseUrl: getCodeExecutionBaseUrl,
+ routeKey: createCodeExecutionRouteKey,
+ profileHeaders: codeExecutionHeaders,
+ downloadQuery: buildCodeEnvDownloadQuery,
+ clearFileCodeEnvRefs,
+ logger,
+ httpAgent: codeServerHttpAgent,
+ httpsAgent: codeServerHttpsAgent,
+});
/**
* Uploads a file to the Code Environment server.
@@ -211,6 +125,7 @@ async function uploadCodeEnvFile({
executionProfile,
bridgeWorkerId,
signal,
+ onUploadStart,
}) {
try {
const form = new FormData();
@@ -237,7 +152,9 @@ async function uploadCodeEnvFile({
...(signal ? { signal } : {}),
};
- const response = await axios.post(`${baseURL}/upload`, form, options);
+ const response = await dispatchFileStorageOperation(onUploadStart, () =>
+ axios.post(`${baseURL}/upload`, form, options),
+ );
/** @type {{ message: string; storage_session_id: string; files: Array<{ fileId: string; filename: string }> }} */
const result = response.data;
diff --git a/api/server/services/Files/Code/crud.spec.js b/api/server/services/Files/Code/crud.spec.js
index 042b6e90a5..b710ee9fd8 100644
--- a/api/server/services/Files/Code/crud.spec.js
+++ b/api/server/services/Files/Code/crud.spec.js
@@ -17,6 +17,8 @@ jest.mock('@librechat/agents', () => ({
* and has its own dedicated `identity.spec.ts` covering the validation
* matrix; this stub just mirrors enough behavior for the surrounding
* crud tests to exercise the upload/download flow. */
+jest.mock('~/models', () => ({ clearFileCodeEnvRefs: jest.fn().mockResolvedValue(true) }));
+
const VALID_KINDS = new Set(['skill', 'agent', 'user']);
const validateIdentity = ({ kind, id, version }, label) => {
if (!kind || !VALID_KINDS.has(kind)) throw new Error(`${label}: invalid kind "${kind}"`);
@@ -33,6 +35,8 @@ jest.mock('@librechat/api', () => {
const http = require('http');
const https = require('https');
return {
+ ...jest.requireActual('../../../../../packages/api/src/files/code/delete'),
+ ...jest.requireActual('../../../../../packages/api/src/files/dispatch'),
appendCodeEnvFile: jest.fn((form, stream, filename) => {
form.append('file', stream, { filename });
}),
@@ -91,6 +95,42 @@ describe('Code CRUD', () => {
getCodeApiAuthHeaders.mockResolvedValue({});
});
+ it('marks upload dispatch after authentication and before a lost HTTP response', async () => {
+ const onUploadStart = jest.fn();
+ mockAxios.post.mockImplementationOnce(async () => {
+ expect(onUploadStart).toHaveBeenCalledTimes(1);
+ throw new Error('transport response lost');
+ });
+ await expect(
+ uploadCodeEnvFile({
+ req: { user: { id: 'user-123' } },
+ stream: Readable.from('data'),
+ filename: 'data.csv',
+ kind: 'user',
+ id: 'user-123',
+ onUploadStart,
+ }),
+ ).rejects.toThrow('transport response lost');
+ expect(onUploadStart).toHaveBeenCalledTimes(1);
+ });
+
+ it('does not mark remote dispatch when upload authentication fails locally', async () => {
+ const onUploadStart = jest.fn();
+ getCodeApiAuthHeaders.mockRejectedValueOnce(new Error('credential lookup unavailable'));
+ await expect(
+ uploadCodeEnvFile({
+ req: { user: { id: 'user-123' } },
+ stream: Readable.from('data'),
+ filename: 'data.csv',
+ kind: 'user',
+ id: 'user-123',
+ onUploadStart,
+ }),
+ ).rejects.toThrow('credential lookup unavailable');
+ expect(onUploadStart).not.toHaveBeenCalled();
+ expect(mockAxios.post).not.toHaveBeenCalled();
+ });
+
describe('getCodeOutputDownloadStream', () => {
/* Code-output downloads always carry `kind: 'user'` + `id: <userId>`
* — codeapi's `sessionAuth` rejects without them post-Phase C. The
@@ -362,7 +402,7 @@ describe('Code CRUD', () => {
Object.assign(new Error('missing'), { response: { status: 404 } }),
);
- await expect(deleteCodeEnvFile(req, file)).resolves.toBeUndefined();
+ await expect(deleteCodeEnvFile(req, file)).resolves.toEqual({ deleted: true });
expect(mockAxios).toHaveBeenCalledTimes(1);
});
diff --git a/api/server/services/Files/Code/process.js b/api/server/services/Files/Code/process.js
index 594e1aba38..405fdda029 100644
--- a/api/server/services/Files/Code/process.js
+++ b/api/server/services/Files/Code/process.js
@@ -3,6 +3,7 @@ const { logger } = require('@librechat/data-schemas');
const { getCodeBaseURL } = require('@librechat/agents');
const {
withTimeout,
+ withFileProvisioning,
getBasePath,
logAxiosError,
hasOfficeHtmlPath,
@@ -57,6 +58,7 @@ const {
const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions');
const { getFiles, updateFile, claimCodeFile, commitCodeFile } = require('~/models');
const { getStrategyFunctions } = require('~/server/services/Files/strategies');
+const { getFileStorage } = require('~/server/services/Files/operations');
const { convertImage } = require('~/server/services/Files/images/convert');
const { getRetentionExpiry } = require('~/server/services/Files/retention');
const { determineFileType } = require('~/server/utils');
@@ -596,6 +598,7 @@ const processCodeOutput = (params) =>
determineFileType,
claimCodeFile,
commitCodeFile,
+ fileStorage: getFileStorage(),
finalizePreview,
hasOfficeHtmlPath,
sanitizeArtifactPath,
@@ -805,82 +808,101 @@ const primeFiles = async (options) => {
const reuploadFile = async () => {
try {
- const { getDownloadStream } = getStrategyFunctions(file.source);
- const { handleFileUpload: uploadCodeEnvFile } = getStrategyFunctions(
- FileSources.execute_code,
- );
- /* Reupload preserves the resource identity from the existing
- * ref so codeapi re-buckets under the same sessionKey shape
- * (skill stays skill, user stays user). Without this, a
- * skill-cache-miss reupload would land in the user bucket
- * and never re-shareable cross-user. */
- const uploaded = await withCodeApiUploadRecovery({
- registry: req.app?.locals?.codeApiUploadRegistry,
- scope: uploadOptions.scope,
- concurrency: uploadOptions.concurrency,
- label: `re-uploading file ${file.file_id} to the code environment`,
- budget: uploadRateLimitBudget,
- signal,
- onWait: (waitMs) =>
- logger.warn(
- `[primeCodeFiles] Rate-limited reupload requestId=${getPrimingCorrelation(req).requestId} ` +
- `runId=${getPrimingCorrelation(req).runId}; retrying in ${waitMs}ms`,
- ),
- openSource: async () => {
- signal?.throwIfAborted();
- return getDownloadStream(options.req, resolveDownloadPath(file), { signal });
- },
- upload: (stream) =>
- uploadCodeEnvFile({
- req: options.req,
- stream,
- filename: sandboxName,
+ const newRef = await withFileProvisioning({
+ file,
+ userId: req.user.id,
+ storage: getFileStorage(),
+ destination: JSON.stringify([
+ 'code',
+ executionRouteKey,
+ sourceRef.kind,
+ sourceRef.id,
+ sourceRef.kind === 'skill' ? sourceRef.version : null,
+ sandboxName,
+ ]),
+ operation: async (_claim, onUploadStart) => {
+ const { getDownloadStream } = getStrategyFunctions(file.source);
+ const { handleFileUpload: uploadCodeEnvFile } = getStrategyFunctions(
+ FileSources.execute_code,
+ );
+ /* Reupload preserves the resource identity from the existing
+ * ref so codeapi re-buckets under the same sessionKey shape
+ * (skill stays skill, user stays user). Without this, a
+ * skill-cache-miss reupload would land in the user bucket
+ * and never re-shareable cross-user. */
+ const uploaded = await withCodeApiUploadRecovery({
+ registry: req.app?.locals?.codeApiUploadRegistry,
+ scope: uploadOptions.scope,
+ concurrency: uploadOptions.concurrency,
+ label: `re-uploading file ${file.file_id} to the code environment`,
+ budget: uploadRateLimitBudget,
+ signal,
+ onWait: (waitMs) =>
+ logger.warn(
+ `[primeCodeFiles] Rate-limited reupload requestId=${getPrimingCorrelation(req).requestId} ` +
+ `runId=${getPrimingCorrelation(req).runId}; retrying in ${waitMs}ms`,
+ ),
+ openSource: async () => {
+ signal?.throwIfAborted();
+ return getDownloadStream(options.req, resolveDownloadPath(file), { signal });
+ },
+ upload: (stream) =>
+ uploadCodeEnvFile({
+ req: options.req,
+ stream,
+ filename: sandboxName,
+ kind: sourceRef.kind,
+ id: sourceRef.id,
+ ...(sourceRef.kind === 'skill' ? { version: sourceRef.version } : {}),
+ codeApiBaseUrl,
+ executionProfile,
+ bridgeWorkerId,
+ signal,
+ onUploadStart,
+ }),
+ });
+
+ /**
+ * Use the FRESH `(storage_session_id, file_id)` from the
+ * reupload response and route it through the persisted record
+ * and the in-memory `files` list. The
+ * original ref captured at the top of this iteration refers
+ * to the old, expired/missing sandbox object — using it here
+ * would silently re-introduce the bug `Graph.sessions`
+ * seeding is supposed to fix.
+ *
+ * `kind`, `id`, `version` survive the round-trip: the
+ * upload preserves the resource identity, only the storage
+ * pointer changes.
+ */
+ const newRef = {
kind: sourceRef.kind,
id: sourceRef.id,
- ...(sourceRef.kind === 'skill' ? { version: sourceRef.version } : {}),
- codeApiBaseUrl,
+ storage_session_id: uploaded.storage_session_id,
+ file_id: uploaded.file_id,
+ sandboxFilename: getUploadedCodeEnvFilename(uploaded, sandboxName),
executionProfile,
- bridgeWorkerId,
- signal,
- }),
- });
-
- /**
- * Use the FRESH `(storage_session_id, file_id)` from the
- * reupload response and route it through the persisted record
- * and the in-memory `files` list. The
- * original ref captured at the top of this iteration refers
- * to the old, expired/missing sandbox object — using it here
- * would silently re-introduce the bug `Graph.sessions`
- * seeding is supposed to fix.
- *
- * `kind`, `id`, `version` survive the round-trip: the
- * upload preserves the resource identity, only the storage
- * pointer changes.
- */
- const newRef = {
- kind: sourceRef.kind,
- id: sourceRef.id,
- storage_session_id: uploaded.storage_session_id,
- file_id: uploaded.file_id,
- sandboxFilename: getUploadedCodeEnvFilename(uploaded, sandboxName),
- executionProfile,
- ...(executionRouteKey !== executionProfile ? { executionRouteKey } : {}),
- ...(sourceRef.kind === 'skill' ? { version: sourceRef.version } : {}),
- };
+ ...(executionRouteKey !== executionProfile ? { executionRouteKey } : {}),
+ ...(sourceRef.kind === 'skill' ? { version: sourceRef.version } : {}),
+ };
- const updatedRefs = mergeCodeEnvRef(file.metadata, newRef);
+ const updatedRefs = mergeCodeEnvRef(file.metadata, newRef);
- await updateFile({
- file_id: file.file_id,
- 'metadata.codeEnvRef': updatedRefs.codeEnvRef,
- [`metadata.codeEnvRefs.${executionRouteKey}`]: newRef,
+ return {
+ value: newRef,
+ commit: {
+ code: {
+ routeKey: executionRouteKey,
+ ref: newRef,
+ legacyRef: updatedRefs.codeEnvRef,
+ },
+ },
+ };
+ },
});
+ file.metadata = { ...file.metadata, ...mergeCodeEnvRef(file.metadata, newRef) };
pushFile(newRef.storage_session_id, newRef.file_id, newRef.sandboxFilename);
- logger.debug(
- `[primeCodeFiles] file=${file.file_id} path=reupload-success ` +
- `oldSession=${session_id} newSession=${newRef.storage_session_id} newFileId=${newRef.file_id}`,
- );
+ logger.debug(`[primeCodeFiles] file=${file.file_id} path=reupload-success`);
} catch (error) {
/* Cancellation is an operation outcome, not a recoverable per-file
* miss. Swallowing it here would keep walking and could dispatch more
diff --git a/api/server/services/Files/Code/process.spec.js b/api/server/services/Files/Code/process.spec.js
index cce7a4ba29..a8a682a2d7 100644
--- a/api/server/services/Files/Code/process.spec.js
+++ b/api/server/services/Files/Code/process.spec.js
@@ -64,6 +64,8 @@ const mockParseSandboxImageChunk = jest.fn((response) => response);
const passthroughWithTimeout = async (promise) => promise;
jest.mock('@librechat/api', () => {
return {
+ withFileProvisioning: jest.requireActual('@librechat/api').withFileProvisioning,
+ confirmFileProvision: jest.requireActual('@librechat/api').confirmFileProvision,
processCodeOutput: jest.requireActual('@librechat/api').processCodeOutput,
createSandboxTextReader: jest.requireActual('@librechat/api').createSandboxTextReader,
prepareCodeOutputBufferForInspection:
@@ -212,10 +214,25 @@ jest.mock('@librechat/agents', () => ({
const mockClaimCodeFile = jest.fn();
const mockCommitCodeFile = jest.fn();
const mockUpdateFile = jest.fn();
+const mockAcquireFileProvision = jest.fn(async () => true);
+const mockReleaseFileProvision = jest.fn(async () => true);
+const mockCommitFileProvision = jest.fn();
+let mockProvisionRecovery;
+jest.mock('~/server/services/Files/operations', () => ({
+ getFileStorage: () => ({
+ recovery: mockProvisionRecovery,
+ acquireFileProvision: (...args) => mockAcquireFileProvision(...args),
+ releaseFileProvision: (...args) => mockReleaseFileProvision(...args),
+ commitFileProvision: (...args) => mockCommitFileProvision(...args),
+ }),
+}));
jest.mock('~/models', () => ({
createFile: jest.fn().mockResolvedValue({}),
getFiles: jest.fn(),
updateFile: mockUpdateFile,
+ acquireFileProvision: (...args) => mockAcquireFileProvision(...args),
+ releaseFileProvision: (...args) => mockReleaseFileProvision(...args),
+ commitFileProvision: (...args) => mockCommitFileProvision(...args),
claimCodeFile: (...args) => mockClaimCodeFile(...args),
commitCodeFile: mockCommitCodeFile,
}));
@@ -298,6 +315,21 @@ describe('Code Process', () => {
beforeEach(() => {
jest.clearAllMocks();
+ mockProvisionRecovery = jest.requireActual('@librechat/api').createFileProvisionRecovery();
+ mockAcquireFileProvision.mockReset().mockResolvedValue(true);
+ mockReleaseFileProvision.mockReset().mockResolvedValue(true);
+ mockCommitFileProvision
+ .mockReset()
+ .mockImplementation(async ({ claim, code, output, releaseOnly }) => {
+ if (releaseOnly) return true;
+ if (output) return mockCommitCodeFile(output.file, output.sourceDispatchedAt);
+ const result = await mockUpdateFile({
+ file_id: claim.file_id,
+ 'metadata.codeEnvRef': code.legacyRef ?? code.ref,
+ [`metadata.codeEnvRefs.${code.routeKey}`]: code.ref,
+ });
+ return result !== null && result !== false;
+ });
fileSizeLimitConfig.value = 20 * 1024 * 1024;
// Default mock: atomic claim returns a new file record (no existing file)
mockClaimCodeFile.mockResolvedValue({
@@ -353,6 +385,60 @@ describe('Code Process', () => {
expect(mockCommitCodeFile).toHaveBeenCalledTimes(1);
});
+ it('does not overwrite a reused output when its storage claim is denied', async () => {
+ mockClaimCodeFile.mockResolvedValue({ file_id: 'existing-output', user: 'user-123' });
+ mockAcquireFileProvision.mockResolvedValueOnce(false);
+ const saveBuffer = jest.fn();
+ getStrategyFunctions.mockReturnValue({ saveBuffer });
+ await processCodeOutput({ ...baseParams, preparedBuffer: Buffer.from('new output') });
+ expect(saveBuffer).not.toHaveBeenCalled();
+ expect(mockCommitCodeFile).not.toHaveBeenCalled();
+ expect(mockReleaseFileProvision).not.toHaveBeenCalled();
+ });
+
+ it('holds regenerated output storage through primary write and full record commit', async () => {
+ mockClaimCodeFile.mockResolvedValue({
+ file_id: 'existing-output',
+ user: 'user-123',
+ messageId: 'original-message',
+ });
+ let finishWrite;
+ const writing = new Promise((resolve) => {
+ finishWrite = resolve;
+ });
+ const saveBuffer = jest.fn(async () => {
+ expect(mockAcquireFileProvision).toHaveBeenCalledTimes(1);
+ await writing;
+ return '/uploads/existing-output.txt';
+ });
+ getStrategyFunctions.mockReturnValue({ saveBuffer });
+ const pending = processCodeOutput({
+ ...baseParams,
+ preparedBuffer: Buffer.from('new output'),
+ });
+ await new Promise((resolve) => setImmediate(resolve));
+ expect(saveBuffer).toHaveBeenCalledTimes(1);
+ expect(mockReleaseFileProvision).not.toHaveBeenCalled();
+ finishWrite();
+ const result = await pending;
+ expect(result.file.file_id).toBe('existing-output');
+ expect(result.file.messageId).toBe(baseParams.messageId);
+ expect(mockCommitCodeFile).toHaveBeenCalledWith(
+ expect.objectContaining({ messageId: 'original-message' }),
+ undefined,
+ );
+ expect(mockCommitFileProvision).toHaveBeenCalledWith(
+ expect.objectContaining({
+ output: expect.objectContaining({
+ file: expect.objectContaining({ file_id: 'existing-output' }),
+ }),
+ }),
+ );
+ expect(mockReleaseFileProvision).toHaveBeenCalledWith(
+ mockAcquireFileProvision.mock.calls[0][0],
+ );
+ });
+
it('enforces a caller-provided aggregate inspection budget while downloading', async () => {
mockAxios.mockResolvedValue({ data: Buffer.from('too large') });
@@ -668,7 +754,7 @@ describe('Code Process', () => {
file_id: 'mock-uuid-1234',
user: 'user-123',
});
- mockCommitCodeFile.mockResolvedValueOnce(false);
+ mockCommitCodeFile.mockResolvedValue(false);
mockAxios.mockResolvedValue({ data: Buffer.alloc(100) });
const result = await processCodeOutput({
@@ -3083,6 +3169,248 @@ describe('Code Process', () => {
return { handleFileUpload, getDownloadStream };
}
+ function claimedFile() {
+ return {
+ file_id: 'claimed-file',
+ user: 'user-123',
+ tenantId: 'tenant-1',
+ filename: 'report.csv',
+ filepath: '/uploads/report.csv',
+ source: 'local',
+ metadata: {
+ codeEnvRef: {
+ kind: 'user',
+ id: 'user-123',
+ storage_session_id: 'OLD_SESSION',
+ file_id: 'OLD_ID',
+ },
+ },
+ };
+ }
+
+ function primeClaimedFile(file) {
+ return primeFiles({
+ req: { user: { id: 'user-123', role: 'USER' } },
+ tool_resources: { execute_code: { file_ids: [file.file_id] } },
+ agentId: 'agent-id',
+ });
+ }
+
+ it('does not open a stream or upload when the durable claim is denied', async () => {
+ const dbFile = claimedFile();
+ const { handleFileUpload, getDownloadStream } = setupReuploadMocks({
+ storage_session_id: 'NEW_SESSION',
+ file_id: 'NEW_ID',
+ });
+ getFiles.mockResolvedValue([dbFile]);
+ mockAcquireFileProvision.mockResolvedValueOnce(false);
+
+ await expect(primeClaimedFile(dbFile)).rejects.toThrow();
+
+ expect(mockAcquireFileProvision).toHaveBeenCalledTimes(1);
+ expect(getDownloadStream).not.toHaveBeenCalled();
+ expect(handleFileUpload).not.toHaveBeenCalled();
+ expect(updateFile).not.toHaveBeenCalled();
+ expect(mockReleaseFileProvision).not.toHaveBeenCalled();
+ });
+
+ it('holds the durable claim through stream upload and pointer persistence', async () => {
+ const dbFile = claimedFile();
+ const { handleFileUpload } = setupReuploadMocks({});
+ getFiles.mockResolvedValue([dbFile]);
+ let finishUpload;
+ let finishWrite;
+ const uploadGate = new Promise((resolve) => {
+ finishUpload = resolve;
+ });
+ const writeGate = new Promise((resolve) => {
+ finishWrite = resolve;
+ });
+ handleFileUpload.mockImplementation(async () => {
+ expect(mockAcquireFileProvision).toHaveBeenCalledTimes(1);
+ await uploadGate;
+ return { storage_session_id: 'NEW_SESSION', file_id: 'NEW_ID' };
+ });
+ updateFile.mockImplementation(async () => {
+ await writeGate;
+ return {};
+ });
+ const pending = primeClaimedFile(dbFile);
+ await new Promise((resolve) => setImmediate(resolve));
+ expect(handleFileUpload).toHaveBeenCalledTimes(1);
+ expect(updateFile).not.toHaveBeenCalled();
+ expect(mockReleaseFileProvision).not.toHaveBeenCalled();
+ finishUpload();
+ await new Promise((resolve) => setImmediate(resolve));
+ expect(updateFile).toHaveBeenCalledTimes(1);
+ expect(mockReleaseFileProvision).not.toHaveBeenCalled();
+ finishWrite();
+ const result = await pending;
+ expect(result.files).toEqual([expect.objectContaining({ id: 'NEW_ID' })]);
+ expect(mockReleaseFileProvision).toHaveBeenCalledTimes(1);
+ expect(mockAcquireFileProvision).toHaveBeenCalledWith(
+ expect.objectContaining({
+ file_id: dbFile.file_id,
+ userId: dbFile.user,
+ tenantId: dbFile.tenantId,
+ token: expect.any(String),
+ }),
+ );
+ expect(mockReleaseFileProvision.mock.calls[0]).toEqual(
+ mockAcquireFileProvision.mock.calls[0],
+ );
+ expect(mockCommitFileProvision).toHaveBeenCalledWith(
+ expect.objectContaining({
+ claim: mockAcquireFileProvision.mock.calls[0][0],
+ code: expect.objectContaining({
+ routeKey: 'default',
+ ref: expect.objectContaining({ file_id: 'NEW_ID' }),
+ }),
+ }),
+ );
+ });
+
+ it('does not seed a remote pointer when its conditional commit is rejected', async () => {
+ const dbFile = claimedFile();
+ setupReuploadMocks({ storage_session_id: 'NEW_SESSION', file_id: 'NEW_ID' });
+ getFiles.mockResolvedValue([dbFile]);
+ mockCommitFileProvision.mockResolvedValue(false);
+ await expect(primeClaimedFile(dbFile)).rejects.toThrow();
+ expect(mockCommitFileProvision).toHaveBeenCalledTimes(2);
+ expect(mockReleaseFileProvision).not.toHaveBeenCalled();
+ });
+
+ it('reuses the original claim and uploaded ref in the next primer invocation', async () => {
+ const dbFile = claimedFile();
+ const { handleFileUpload } = setupReuploadMocks({
+ storage_session_id: 'NEW_SESSION',
+ file_id: 'NEW_ID',
+ });
+ getFiles.mockResolvedValue([dbFile]);
+ mockCommitFileProvision.mockResolvedValueOnce(false).mockResolvedValueOnce(false);
+ await expect(primeClaimedFile(dbFile)).rejects.toThrow();
+ const originalClaim = mockAcquireFileProvision.mock.calls[0][0];
+ const result = await primeClaimedFile(dbFile);
+ expect(result.files).toEqual([expect.objectContaining({ id: 'NEW_ID' })]);
+ expect(handleFileUpload).toHaveBeenCalledTimes(1);
+ expect(mockAcquireFileProvision).toHaveBeenCalledTimes(1);
+ expect(mockCommitFileProvision).toHaveBeenLastCalledWith(
+ expect.objectContaining({ claim: originalClaim }),
+ );
+ expect(mockReleaseFileProvision).toHaveBeenCalledWith(originalClaim);
+ });
+
+ it.each(['primer', 'callback'])(
+ 'recovers a %s upload through the other producer while provisioning an independent file',
+ async (first) => {
+ const { createProvisionFilesCallback } = jest.requireActual('@librechat/api');
+ const { getFileStorage } = require('~/server/services/Files/operations');
+ const dbFile = claimedFile();
+ const independent = {
+ ...claimedFile(),
+ file_id: 'independent-file',
+ filename: 'independent.csv',
+ };
+ const { handleFileUpload } = setupReuploadMocks({
+ storage_session_id: 'NEW_SESSION',
+ file_id: 'NEW_ID',
+ filename: 'stored.csv',
+ });
+ handleFileUpload.mockImplementation(async ({ filename }) => ({
+ storage_session_id: 'NEW_SESSION',
+ file_id: filename === dbFile.filename ? 'NEW_ID' : 'INDEPENDENT_ID',
+ filename: filename === dbFile.filename ? 'stored.csv' : filename,
+ }));
+ const req = { user: { id: 'user-123', role: 'USER' } };
+ const provisionToCodeEnv = jest.fn(async ({ file, sandboxFilename }) => {
+ const ref = {
+ kind: 'user',
+ id: req.user.id,
+ storage_session_id: 'NEW_SESSION',
+ file_id: file === dbFile ? 'NEW_ID' : 'INDEPENDENT_ID',
+ sandboxFilename: file === dbFile ? 'stored.csv' : sandboxFilename,
+ };
+ return {
+ referenceSet: { codeEnvRefs: { default: ref } },
+ refUpdate: { file_id: file.file_id, routeKey: 'default', ref },
+ sandboxFilename: ref.sandboxFilename,
+ };
+ });
+ const callback = (files) => {
+ const provisionState = {
+ codeEnvFiles: files,
+ vectorDBFiles: [],
+ aliveFileIds: new Set(),
+ agentScopedFileIds: new Set(),
+ };
+ return createProvisionFilesCallback({
+ req,
+ fileStorage: getFileStorage(),
+ agentToolContexts: new Map([['agent-id', { provisionState }]]),
+ provisionToCodeEnv,
+ provisionToVectorDB: jest.fn(),
+ updateFile: jest.fn(),
+ updateCodeEnvRef: jest.fn(),
+ addEmbeddedEntity: jest.fn(),
+ })(['execute_code'], 'agent-id');
+ };
+ const primer = (files) => {
+ getFiles.mockResolvedValue(files);
+ return primeFiles({
+ req,
+ tool_resources: { execute_code: { file_ids: files.map((file) => file.file_id) } },
+ agentId: 'agent-id',
+ });
+ };
+ mockCommitFileProvision.mockResolvedValueOnce(false).mockResolvedValueOnce(false);
+ await expect(first === 'primer' ? primer([dbFile]) : callback([dbFile])).rejects.toThrow();
+ const claim = mockAcquireFileProvision.mock.calls[0][0];
+ const result =
+ first === 'primer'
+ ? await callback([dbFile, independent])
+ : (await primer([dbFile, independent])).files;
+ expect(result).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ id: 'NEW_ID', name: 'stored.csv' }),
+ expect.objectContaining({ id: 'INDEPENDENT_ID', name: 'independent.csv' }),
+ ]),
+ );
+ expect(result).toHaveLength(2);
+ expect(handleFileUpload).toHaveBeenCalledTimes(1);
+ expect(provisionToCodeEnv).toHaveBeenCalledTimes(1);
+ expect(mockAcquireFileProvision).toHaveBeenCalledTimes(2);
+ expect(mockReleaseFileProvision).toHaveBeenCalledWith(claim);
+ expect(dbFile.metadata.codeEnvRefs.default).toMatchObject({
+ file_id: 'NEW_ID',
+ sandboxFilename: 'stored.csv',
+ });
+ },
+ );
+
+ it.each(['upload', 'persist'])(
+ 'handles the durable claim after a %s failure',
+ async (failure) => {
+ const dbFile = claimedFile();
+ const { handleFileUpload } = setupReuploadMocks({
+ storage_session_id: 'NEW_SESSION',
+ file_id: 'NEW_ID',
+ });
+ getFiles.mockResolvedValue([dbFile]);
+ if (failure === 'upload')
+ handleFileUpload.mockRejectedValueOnce(new Error('upload unavailable'));
+ else updateFile.mockRejectedValue(new Error('database unavailable'));
+ await expect(primeClaimedFile(dbFile)).rejects.toThrow();
+ if (failure === 'upload') {
+ expect(mockReleaseFileProvision).toHaveBeenCalledTimes(1);
+ expect(mockReleaseFileProvision.mock.calls[0]).toEqual(
+ mockAcquireFileProvision.mock.calls[0],
+ );
+ } else {
+ expect(mockReleaseFileProvision).not.toHaveBeenCalled();
+ }
+ },
+ );
+
it('recovers a throttled stale-file reupload with a fresh stream', async () => {
const controller = new AbortController();
const dbFile = {
diff --git a/api/server/services/Files/VectorDB/crud.js b/api/server/services/Files/VectorDB/crud.js
index 9f43f0e4ae..359058dafa 100644
--- a/api/server/services/Files/VectorDB/crud.js
+++ b/api/server/services/Files/VectorDB/crud.js
@@ -3,50 +3,19 @@ const axios = require('axios');
const FormData = require('form-data');
const { logger } = require('@librechat/data-schemas');
const { FileSources } = require('librechat-data-provider');
-const { logAxiosError, generateShortLivedToken } = require('@librechat/api');
+const {
+ logAxiosError,
+ generateShortLivedToken,
+ createVectorFileDeleter,
+ dispatchFileStorageOperation,
+} = require('@librechat/api');
-/**
- * Deletes a file from the vector database. This function takes a file object, constructs the full path, and
- * verifies the path's validity before deleting the file. If the path is invalid, an error is thrown.
- *
- * @param {ServerRequest} req - The request object from Express.
- * @param {MongoFile} file - The file object to be deleted. It should have a `filepath` property that is
- * a string representing the path of the file relative to the publicPath.
- *
- * @returns {Promise<void>}
- * A promise that resolves when the file has been successfully deleted, or throws an error if the
- * file path is invalid or if there is an error in deletion.
- */
-const deleteVectors = async (req, file) => {
- if (!file.embedded || !process.env.RAG_API_URL) {
- return;
- }
- try {
- const jwtToken = generateShortLivedToken(req.user.id);
-
- return await axios.delete(`${process.env.RAG_API_URL}/documents`, {
- headers: {
- Authorization: `Bearer ${jwtToken}`,
- 'Content-Type': 'application/json',
- accept: 'application/json',
- },
- data: [file.file_id],
- });
- } catch (error) {
- logAxiosError({
- error,
- message: 'Error deleting vectors',
- });
- if (
- error.response &&
- error.response.status !== 404 &&
- (error.response.status < 200 || error.response.status >= 300)
- ) {
- logger.warn('Error deleting vectors, file will not be deleted');
- throw new Error(error.message || 'An error occurred during file deletion.');
- }
- }
-};
+const deleteVectors = (req, file) =>
+ createVectorFileDeleter({
+ baseUrl: process.env.RAG_API_URL,
+ httpClient: axios,
+ generateToken: generateShortLivedToken,
+ })(req, file);
/**
* Uploads a file to the configured Vector database
@@ -64,7 +33,15 @@ const deleteVectors = async (req, file) => {
* - filepath: The path where the file is saved.
* - bytes: The size of the file in bytes.
*/
-async function uploadVectors({ req, file, file_id, entity_id, storageMetadata, signal }) {
+async function uploadVectors({
+ req,
+ file,
+ file_id,
+ entity_id,
+ storageMetadata,
+ signal,
+ onUploadStart,
+}) {
if (!process.env.RAG_API_URL) {
throw new Error('RAG_API_URL not defined');
}
@@ -85,14 +62,16 @@ async function uploadVectors({ req, file, file_id, entity_id, storageMetadata, s
const formHeaders = formData.getHeaders();
- const response = await axios.post(`${process.env.RAG_API_URL}/embed`, formData, {
- ...(signal ? { signal } : {}),
- headers: {
- Authorization: `Bearer ${jwtToken}`,
- accept: 'application/json',
- ...formHeaders,
- },
- });
+ const response = await dispatchFileStorageOperation(onUploadStart, () =>
+ axios.post(`${process.env.RAG_API_URL}/embed`, formData, {
+ ...(signal ? { signal } : {}),
+ headers: {
+ Authorization: `Bearer ${jwtToken}`,
+ accept: 'application/json',
+ ...formHeaders,
+ },
+ }),
+ );
const responseData = response.data;
logger.debug('Response from embedding file', responseData);
@@ -116,7 +95,7 @@ async function uploadVectors({ req, file, file_id, entity_id, storageMetadata, s
error,
message: 'Error uploading vectors',
});
- throw new Error(error.message || 'An error occurred during file upload.');
+ throw new Error(error.message || 'An error occurred during file upload.', { cause: error });
}
}
diff --git a/api/server/services/Files/operations.js b/api/server/services/Files/operations.js
new file mode 100644
index 0000000000..9db901fea7
--- /dev/null
+++ b/api/server/services/Files/operations.js
@@ -0,0 +1,15 @@
+const { createFileProvisionRecovery } = require('@librechat/api');
+const db = require('~/models');
+
+let storage;
+const getFileStorage = () =>
+ (storage ??= {
+ recovery: createFileProvisionRecovery(),
+ acquireFileProvision: db.acquireFileProvision,
+ releaseFileProvision: db.releaseFileProvision,
+ commitFileProvision: db.commitFileProvision,
+ acquireFileDeletion: db.acquireFileDeletion,
+ releaseFileDeletion: db.releaseFileDeletion,
+ });
+
+module.exports = { getFileStorage };
diff --git a/api/server/services/Files/process.integration.spec.js b/api/server/services/Files/process.integration.spec.js
index ee47e0d109..80b679652e 100644
--- a/api/server/services/Files/process.integration.spec.js
+++ b/api/server/services/Files/process.integration.spec.js
@@ -30,6 +30,7 @@ jest.mock('@librechat/agents', () => ({
}));
jest.mock('@librechat/api', () => ({
+ ...jest.requireActual('../../../../packages/api/src/files/finalize'),
sanitizeFilename: jest.fn((n) => n),
parseText: jest.fn().mockResolvedValue({ text: '', bytes: 0 }),
processAudioFile: jest.fn(),
diff --git a/api/server/services/Files/process.js b/api/server/services/Files/process.js
index 160fd18cd0..0dc3a0bfe3 100644
--- a/api/server/services/Files/process.js
+++ b/api/server/services/Files/process.js
@@ -30,6 +30,9 @@ const {
const { logger, runAsSystem } = require('@librechat/data-schemas');
const {
sanitizeFilename,
+ isMissingStorageError,
+ createDeleteFileWithSecondaryStorage,
+ finalizeFileDeletion,
parseText,
processAudioFile,
extractInspectableFileText,
@@ -73,6 +76,7 @@ const { getStrategyFunctions } = require('./strategies');
const { determineFileType } = require('~/server/utils');
const { STTService } = require('./Audio/STTService');
const db = require('~/models');
+const { getFileStorage } = require('~/server/services/Files/operations');
/**
* Creates a modular file upload wrapper that ensures filename sanitization
@@ -97,20 +101,6 @@ const createSanitizedUploadWrapper = (uploadFunction) => {
};
};
-const hasCodeEnvRef = (file) =>
- file?.metadata?.codeEnvRef != null || file?.metadata?.codeEnvRefs != null;
-
-const isMissingStorageError = (err) => {
- const code = err?.code ?? err?.status ?? err?.statusCode ?? err?.response?.status;
- if ([404, '404', 'ENOENT', 'NoSuchKey', 'NotFound', 'ResourceNotFound'].includes(code)) {
- return true;
- }
-
- return /(?:file|object|blob|key|resource) (?:not found|does not exist)|no such (?:file|key)/i.test(
- String(err?.message ?? ''),
- );
-};
-
/**
* Enqueues the delete operation to the leaky bucket queue if necessary, or adds it directly to promises.
*
@@ -191,35 +181,6 @@ const getDeleteMethod = ({ source, deletionMethods }) => {
return deleteFile;
};
-const createDeleteFileWithSecondaryStorage = ({ source, deleteFile, deletionMethods }) => {
- return async (req, file, openai) => {
- const secondaryDeleteMethods = [];
- if (file.embedded === true && source !== FileSources.vectordb) {
- secondaryDeleteMethods.push(
- getDeleteMethod({ source: FileSources.vectordb, deletionMethods }),
- );
- }
- if (hasCodeEnvRef(file) && source !== FileSources.execute_code) {
- secondaryDeleteMethods.push(
- getDeleteMethod({ source: FileSources.execute_code, deletionMethods }),
- );
- }
-
- try {
- await deleteFile(req, file, openai);
- } catch (err) {
- if (!isMissingStorageError(err)) {
- throw err;
- }
- logger.warn('Primary file storage was already missing during delete', err);
- }
-
- await Promise.all(
- secondaryDeleteMethods.map((secondaryDeleteFile) => secondaryDeleteFile(req, file)),
- );
- };
-};
-
const resolveUploadDeleteFile = (source) =>
createStoredUploadDelete({
source,
@@ -307,7 +268,15 @@ const processDeleteRequest = async ({ req, files }) => {
enqueueDeleteOperation({
req,
file,
- deleteFile: createDeleteFileWithSecondaryStorage({ source, deleteFile, deletionMethods }),
+ deleteFile: createDeleteFileWithSecondaryStorage({
+ source,
+ deleteFile,
+ getDeleteFile: (src) => getDeleteMethod({ source: src, deletionMethods }),
+ clearFileEmbeddings: db.clearFileEmbeddings,
+ isFileStorageClean: db.isFileStorageClean,
+ fileStorage: getFileStorage(),
+ logger,
+ }),
promises,
resolvedFileIds,
failedFileIds,
@@ -316,33 +285,19 @@ const processDeleteRequest = async ({ req, files }) => {
}
await Promise.allSettled(promises);
- const deletedFileIds = [...resolvedFileIds];
- let metadataDeletedFileIds = deletedFileIds;
- if (deletedFileIds.length > 0) {
- try {
- await db.deleteFiles(deletedFileIds);
- } catch (error) {
- logger.error('Error deleting file metadata after storage deletion', error);
- deletedFileIds.forEach((fileId) => failedFileIds.add(fileId));
- metadataDeletedFileIds = [];
- throw error;
- }
- /* The only place a delete removes agent references, and it runs after the metadata delete
- succeeded: a file that kept its storage, its chunks or its record keeps its references too,
- so the agent it was removed from can be asked again (see issue #12776). */
- if (metadataDeletedFileIds.length > 0) {
- try {
- await db.removeAgentResourceFilesFromAllAgents({ file_ids: metadataDeletedFileIds });
- } catch (error) {
- logger.error('Error cleaning up orphaned agent file references', error);
- }
- }
- }
-
- return {
- deletedFileIds: metadataDeletedFileIds,
- failedFileIds: [...failedFileIds],
- };
+ return finalizeFileDeletion(
+ {
+ deletedFileIds: [...resolvedFileIds],
+ failedFileIds: [...failedFileIds],
+ files,
+ userId: req.user.id,
+ },
+ {
+ deleteFiles: db.deleteFiles,
+ removeAgentResourceFilesFromAllAgents: db.removeAgentResourceFilesFromAllAgents,
+ logger,
+ },
+ );
};
/**
diff --git a/api/server/services/Files/process.spec.js b/api/server/services/Files/process.spec.js
index 1454ac5dbb..1f140e34ef 100644
--- a/api/server/services/Files/process.spec.js
+++ b/api/server/services/Files/process.spec.js
@@ -121,6 +121,9 @@ jest.mock('@librechat/api', () => {
},
);
return {
+ ...jest.requireActual('../../../../packages/api/src/files/storage'),
+ ...jest.requireActual('../../../../packages/api/src/files/operations'),
+ ...jest.requireActual('../../../../packages/api/src/files/finalize'),
sanitizeFilename: jest.fn((n) => n),
/** Grants both; these specs vary the capability set, not the role. */
resolveToolRoleGrants: jest.fn(async () => ({ runCode: true, fileSearch: true })),
@@ -242,6 +245,10 @@ jest.mock('~/models', () => ({
createFile: jest.fn().mockResolvedValue({ file_id: 'created-file-id' }),
updateFileUsage: jest.fn(),
deleteFiles: jest.fn(),
+ isFileStorageClean: jest.fn().mockResolvedValue(true),
+ acquireFileDeletion: jest.fn().mockResolvedValue({ resumed: false }),
+ releaseFileDeletion: jest.fn().mockResolvedValue(true),
+ clearFileEmbeddings: jest.fn().mockResolvedValue(true),
findFileById: jest.fn(),
getConvo: jest.fn(),
getExpiredFiles: jest.fn(),
@@ -2653,20 +2660,31 @@ describe('processFileURL', () => {
describe('processDeleteRequest', () => {
beforeEach(() => {
jest.clearAllMocks();
+ db.clearFileEmbeddings.mockResolvedValue(true);
+ db.deleteFiles.mockImplementation(async (ids) => ({
+ deletedCount: ids.length,
+ deletedFileIds: ids,
+ failedFileIds: [],
+ }));
});
it('removes metadata when backing storage is already missing', async () => {
const missingError = Object.assign(new Error('no such file'), { code: 'ENOENT' });
const deleteFile = jest.fn().mockRejectedValue(missingError);
getStrategyFunctions.mockReturnValue({ deleteFile });
- db.deleteFiles.mockResolvedValue({ deletedCount: 1 });
+ db.deleteFiles.mockImplementation(async (ids) => ({
+ deletedCount: ids.length,
+ deletedFileIds: ids,
+ failedFileIds: [],
+ }));
+ const req = {
+ body: {},
+ config: { fileConfig: { retainedFileCleanup: true } },
+ user: { id: 'user-123', tenantId: 'tenant-a' },
+ };
const result = await processDeleteRequest({
- req: {
- body: {},
- config: {},
- user: { id: 'user-123', tenantId: 'tenant-a' },
- },
+ req,
files: [
{
file_id: 'expired-file',
@@ -2676,7 +2694,11 @@ describe('processDeleteRequest', () => {
],
});
- expect(db.deleteFiles).toHaveBeenCalledWith(['expired-file']);
+ expect(db.deleteFiles).toHaveBeenCalledWith(
+ ['expired-file'],
+ undefined,
+ expect.objectContaining({ userId: req.user.id }),
+ );
expect(result).toEqual({ deletedFileIds: ['expired-file'], failedFileIds: [] });
});
@@ -2687,7 +2709,7 @@ describe('processDeleteRequest', () => {
const result = await processDeleteRequest({
req: {
body: {},
- config: {},
+ config: { fileConfig: { retainedFileCleanup: true } },
user: { id: 'user-123', tenantId: 'tenant-a' },
},
files: [
@@ -2704,18 +2726,19 @@ describe('processDeleteRequest', () => {
});
it('throws metadata delete failures after storage deletion succeeds', async () => {
- const deleteFile = jest.fn().mockResolvedValue(undefined);
+ const deleteFile = jest.fn().mockResolvedValue({ deleted: true });
const metadataError = new Error('mongo unavailable');
getStrategyFunctions.mockReturnValue({ deleteFile });
db.deleteFiles.mockRejectedValue(metadataError);
+ const req = {
+ body: {},
+ config: { fileConfig: { retainedFileCleanup: true } },
+ user: { id: 'user-123', tenantId: 'tenant-a' },
+ };
await expect(
processDeleteRequest({
- req: {
- body: {},
- config: {},
- user: { id: 'user-123', tenantId: 'tenant-a' },
- },
+ req,
files: [
{
file_id: 'expired-file',
@@ -2726,22 +2749,30 @@ describe('processDeleteRequest', () => {
}),
).rejects.toThrow('mongo unavailable');
- expect(db.deleteFiles).toHaveBeenCalledWith(['expired-file']);
+ expect(db.deleteFiles).toHaveBeenCalledWith(
+ ['expired-file'],
+ undefined,
+ expect.objectContaining({ userId: req.user.id }),
+ );
expect(db.removeAgentResourceFilesFromAllAgents).not.toHaveBeenCalled();
});
it('deletes vector storage before removing embedded file metadata', async () => {
- const primaryDelete = jest.fn().mockResolvedValue(undefined);
- const vectorDelete = jest.fn().mockResolvedValue(undefined);
+ const primaryDelete = jest.fn().mockResolvedValue({ deleted: true });
+ const vectorDelete = jest.fn().mockResolvedValue({ deleted: true });
getStrategyFunctions.mockImplementation((source) =>
source === FileSources.vectordb
? { deleteFile: vectorDelete }
: { deleteFile: primaryDelete },
);
- db.deleteFiles.mockResolvedValue({ deletedCount: 1 });
+ db.deleteFiles.mockImplementation(async (ids) => ({
+ deletedCount: ids.length,
+ deletedFileIds: ids,
+ failedFileIds: [],
+ }));
const req = {
body: {},
- config: {},
+ config: { fileConfig: { retainedFileCleanup: true } },
user: { id: 'user-123', tenantId: 'tenant-a' },
};
const file = {
@@ -2755,12 +2786,39 @@ describe('processDeleteRequest', () => {
expect(primaryDelete).toHaveBeenCalledWith(req, file, undefined);
expect(vectorDelete).toHaveBeenCalledWith(req, file);
- expect(db.deleteFiles).toHaveBeenCalledWith(['embedded-file']);
+ expect(db.deleteFiles).toHaveBeenCalledWith(
+ ['embedded-file'],
+ undefined,
+ expect.objectContaining({ userId: req.user.id }),
+ );
expect(result).toEqual({ deletedFileIds: ['embedded-file'], failedFileIds: [] });
});
- it('keeps embedded file metadata when vector deletion fails', async () => {
- const primaryDelete = jest.fn().mockResolvedValue(undefined);
+ it('retains truthful primary-vector state when metadata deletion subsequently fails', async () => {
+ const vectorDelete = jest.fn().mockResolvedValue({ deleted: true });
+ getStrategyFunctions.mockReturnValue({ deleteFile: vectorDelete });
+ db.deleteFiles.mockRejectedValue(new Error('mongo unavailable'));
+ const req = {
+ body: {},
+ config: { fileConfig: { retainedFileCleanup: true } },
+ user: { id: 'user-123', tenantId: 'tenant-a' },
+ };
+ const file = {
+ file_id: 'vector-primary',
+ source: FileSources.vectordb,
+ embedded: true,
+ metadata: { embeddedEntities: ['namespace'] },
+ };
+ await expect(processDeleteRequest({ req, files: [file] })).rejects.toThrow('mongo unavailable');
+ expect(vectorDelete).toHaveBeenCalledTimes(1);
+ expect(db.clearFileEmbeddings).toHaveBeenCalledTimes(1);
+ expect(file.embedded).toBe(false);
+ expect(file.metadata).toEqual({ searchDestination: true });
+ expect(db.removeAgentResourceFilesFromAllAgents).not.toHaveBeenCalled();
+ });
+
+ it('keeps primary storage and metadata when vector deletion fails', async () => {
+ const primaryDelete = jest.fn().mockResolvedValue({ deleted: true });
const vectorDelete = jest.fn().mockRejectedValue(new Error('rag unavailable'));
getStrategyFunctions.mockImplementation((source) =>
source === FileSources.vectordb
@@ -2769,7 +2827,7 @@ describe('processDeleteRequest', () => {
);
const req = {
body: {},
- config: {},
+ config: { fileConfig: { retainedFileCleanup: true } },
user: { id: 'user-123', tenantId: 'tenant-a' },
};
const file = {
@@ -2781,18 +2839,126 @@ describe('processDeleteRequest', () => {
const result = await processDeleteRequest({ req, files: [file] });
- expect(primaryDelete).toHaveBeenCalledWith(req, file, undefined);
+ expect(primaryDelete).not.toHaveBeenCalled();
expect(vectorDelete).toHaveBeenCalledWith(req, file);
expect(db.deleteFiles).not.toHaveBeenCalled();
expect(result).toEqual({ deletedFileIds: [], failedFileIds: ['embedded-file'] });
});
+ it('retains primary storage, metadata and agent references through a partial cleanup retry', async () => {
+ const primaryDelete = jest.fn().mockResolvedValue({ deleted: true });
+ const vectorDelete = jest
+ .fn()
+ .mockResolvedValueOnce({ deleted: true })
+ .mockRejectedValueOnce(Object.assign(new Error('already gone'), { status: 404 }));
+ const codeDelete = jest
+ .fn()
+ .mockRejectedValueOnce(
+ Object.assign(new Error('code unavailable'), { response: { status: 500 } }),
+ )
+ .mockResolvedValueOnce({ deleted: true });
+ const deleteMethods = {
+ [FileSources.vectordb]: vectorDelete,
+ [FileSources.execute_code]: codeDelete,
+ };
+ getStrategyFunctions.mockImplementation((source) => ({
+ deleteFile: deleteMethods[source] ?? primaryDelete,
+ }));
+ const req = {
+ body: { agent_id: 'agent_1', tool_resource: 'file_search' },
+ config: { fileConfig: { retainedFileCleanup: true } },
+ user: { id: 'user-123', tenantId: 'tenant-a' },
+ };
+ const file = {
+ file_id: 'retry-file',
+ filepath: '/uploads/retry.png',
+ source: FileSources.local,
+ embedded: true,
+ metadata: { codeEnvRefs: { default: { file_id: 'code-copy' } } },
+ };
+
+ expect(await processDeleteRequest({ req, files: [file] })).toEqual({
+ deletedFileIds: [],
+ failedFileIds: ['retry-file'],
+ });
+ expect(primaryDelete).not.toHaveBeenCalled();
+ expect(db.deleteFiles).not.toHaveBeenCalled();
+ expect(db.removeAgentResourceFilesFromAllAgents).not.toHaveBeenCalled();
+
+ expect(file.embedded).toBe(false);
+ expect(file.metadata.searchDestination).toBe(true);
+
+ db.deleteFiles.mockImplementation(async (ids) => ({
+ deletedCount: ids.length,
+ deletedFileIds: ids,
+ failedFileIds: [],
+ }));
+ expect(await processDeleteRequest({ req, files: [file] })).toEqual({
+ deletedFileIds: ['retry-file'],
+ failedFileIds: [],
+ });
+ expect(primaryDelete).toHaveBeenCalledTimes(1);
+ expect(primaryDelete).toHaveBeenCalledWith(req, file, undefined);
+ expect(vectorDelete).toHaveBeenCalledTimes(1);
+ expect(db.clearFileEmbeddings).toHaveBeenCalledTimes(1);
+ expect(codeDelete).toHaveBeenCalledTimes(2);
+ expect(db.deleteFiles).toHaveBeenCalledWith(
+ ['retry-file'],
+ undefined,
+ expect.objectContaining({ userId: req.user.id }),
+ );
+ expect(db.removeAgentResourceFilesFromAllAgents).toHaveBeenCalledWith({
+ file_ids: ['retry-file'],
+ });
+ });
+
+ it.each(['primary', 'code', 'persistence'])(
+ 'keeps the retry anchor and agent association after %s failure',
+ async (failure) => {
+ const primaryDelete = jest.fn().mockResolvedValue({ deleted: true });
+ const codeDelete = jest.fn().mockResolvedValue({ deleted: true });
+ const vectorDelete = jest.fn().mockResolvedValue({ deleted: true });
+ const methods = {
+ [FileSources.vectordb]: vectorDelete,
+ [FileSources.execute_code]: codeDelete,
+ };
+ getStrategyFunctions.mockImplementation((source) => ({
+ deleteFile: methods[source] ?? primaryDelete,
+ }));
+ if (failure === 'persistence') {
+ db.clearFileEmbeddings.mockRejectedValue(new Error('resource not found'));
+ } else {
+ (failure === 'primary' ? primaryDelete : codeDelete).mockRejectedValue(new Error('denied'));
+ }
+ const file = {
+ file_id: 'retained-agent-file',
+ source: FileSources.local,
+ embedded: true,
+ metadata: { embeddedEntities: ['agent-a'], codeEnvRef: { file_id: 'code-copy' } },
+ };
+ const req = {
+ body: { agent_id: 'agent-a', tool_resource: 'file_search' },
+ config: { fileConfig: { retainedFileCleanup: true } },
+ user: { id: 'owner' },
+ };
+ expect(await processDeleteRequest({ req, files: [file] })).toEqual({
+ deletedFileIds: [],
+ failedFileIds: ['retained-agent-file'],
+ });
+ expect(db.deleteFiles).not.toHaveBeenCalled();
+ expect(db.removeAgentResourceFilesFromAllAgents).not.toHaveBeenCalled();
+ expect(file.embedded).toBe(failure === 'persistence');
+ expect(file.metadata.codeEnvRef.file_id).toBe('code-copy');
+ if (failure !== 'primary') expect(primaryDelete).not.toHaveBeenCalled();
+ },
+ );
+
it('keeps a failed agent file attached so the delete can be retried', async () => {
const deleteFile = jest.fn().mockRejectedValue(new Error('rag unavailable'));
getStrategyFunctions.mockReturnValue({ deleteFile });
const req = {
body: { agent_id: 'agent_1', tool_resource: 'file_search' },
- config: {},
+ config: { fileConfig: { retainedFileCleanup: true } },
user: { id: 'user-123', tenantId: 'tenant-a' },
};
@@ -2823,10 +2989,14 @@ describe('processDeleteRequest', () => {
: Promise.resolve(undefined),
),
});
- db.deleteFiles.mockResolvedValue({ deletedCount: 1 });
+ db.deleteFiles.mockImplementation(async (ids) => ({
+ deletedCount: ids.length,
+ deletedFileIds: ids,
+ failedFileIds: [],
+ }));
const req = {
body: { agent_id: 'agent_1', tool_resource: 'file_search' },
- config: {},
+ config: { fileConfig: { retainedFileCleanup: true } },
user: { id: 'user-123', tenantId: 'tenant-a' },
};
@@ -2844,12 +3014,32 @@ describe('processDeleteRequest', () => {
});
});
+ it('finalizes successful file references while retaining a concurrent replacement', async () => {
+ getStrategyFunctions.mockReturnValue({
+ deleteFile: jest.fn().mockResolvedValue({ deleted: true }),
+ });
+ db.deleteFiles.mockResolvedValue({
+ deletedCount: 1,
+ deletedFileIds: ['gone'],
+ failedFileIds: ['kept'],
+ });
+ const req = { body: {}, config: {}, user: { id: 'user-123' } };
+ const files = ['gone', 'kept'].map((file_id) => ({ file_id, source: FileSources.local }));
+ expect(await processDeleteRequest({ req, files })).toEqual({
+ deletedFileIds: ['gone'],
+ failedFileIds: ['kept'],
+ });
+ expect(db.removeAgentResourceFilesFromAllAgents).toHaveBeenCalledWith({ file_ids: ['gone'] });
+ });
+
it('keeps agent references when the metadata delete fails', async () => {
- getStrategyFunctions.mockReturnValue({ deleteFile: jest.fn().mockResolvedValue(undefined) });
+ getStrategyFunctions.mockReturnValue({
+ deleteFile: jest.fn().mockResolvedValue({ deleted: true }),
+ });
db.deleteFiles.mockRejectedValue(new Error('mongo unavailable'));
const req = {
body: { agent_id: 'agent_1', tool_resource: 'file_search' },
- config: {},
+ config: { fileConfig: { retainedFileCleanup: true } },
user: { id: 'user-123', tenantId: 'tenant-a' },
};
@@ -2870,9 +3060,9 @@ describe('processDeleteRequest', () => {
expect(db.removeAgentResourceFilesFromAllAgents).not.toHaveBeenCalled();
});
- it('does not delete vector storage when primary embedded file deletion fails', async () => {
+ it('retains metadata when primary deletion fails after secondary cleanup', async () => {
const primaryDelete = jest.fn().mockRejectedValue(new Error('permission denied'));
- const vectorDelete = jest.fn().mockResolvedValue(undefined);
+ const vectorDelete = jest.fn().mockResolvedValue({ deleted: true });
getStrategyFunctions.mockImplementation((source) =>
source === FileSources.vectordb
? { deleteFile: vectorDelete }
@@ -2880,7 +3070,7 @@ describe('processDeleteRequest', () => {
);
const req = {
body: {},
- config: {},
+ config: { fileConfig: { retainedFileCleanup: true } },
user: { id: 'user-123', tenantId: 'tenant-a' },
};
const file = {
@@ -2893,7 +3083,7 @@ describe('processDeleteRequest', () => {
const result = await processDeleteRequest({ req, files: [file] });
expect(primaryDelete).toHaveBeenCalledWith(req, file, undefined);
- expect(vectorDelete).not.toHaveBeenCalled();
+ expect(vectorDelete).toHaveBeenCalledWith(req, file);
expect(db.deleteFiles).not.toHaveBeenCalled();
expect(result).toEqual({ deletedFileIds: [], failedFileIds: ['embedded-file'] });
});
@@ -2901,16 +3091,20 @@ describe('processDeleteRequest', () => {
it('still deletes vector storage when primary embedded file storage is already missing', async () => {
const missingError = Object.assign(new Error('no such file'), { code: 'ENOENT' });
const primaryDelete = jest.fn().mockRejectedValue(missingError);
- const vectorDelete = jest.fn().mockResolvedValue(undefined);
+ const vectorDelete = jest.fn().mockResolvedValue({ deleted: true });
getStrategyFunctions.mockImplementation((source) =>
source === FileSources.vectordb
? { deleteFile: vectorDelete }
: { deleteFile: primaryDelete },
);
- db.deleteFiles.mockResolvedValue({ deletedCount: 1 });
+ db.deleteFiles.mockImplementation(async (ids) => ({
+ deletedCount: ids.length,
+ deletedFileIds: ids,
+ failedFileIds: [],
+ }));
const req = {
body: {},
- config: {},
+ config: { fileConfig: { retainedFileCleanup: true } },
user: { id: 'user-123', tenantId: 'tenant-a' },
};
const file = {
@@ -2924,22 +3118,30 @@ describe('processDeleteRequest', () => {
expect(primaryDelete).toHaveBeenCalledWith(req, file, undefined);
expect(vectorDelete).toHaveBeenCalledWith(req, file);
- expect(db.deleteFiles).toHaveBeenCalledWith(['embedded-file']);
+ expect(db.deleteFiles).toHaveBeenCalledWith(
+ ['embedded-file'],
+ undefined,
+ expect.objectContaining({ userId: req.user.id }),
+ );
expect(result).toEqual({ deletedFileIds: ['embedded-file'], failedFileIds: [] });
});
it('deletes code environment storage before removing code resource file metadata', async () => {
- const primaryDelete = jest.fn().mockResolvedValue(undefined);
- const codeDelete = jest.fn().mockResolvedValue(undefined);
+ const primaryDelete = jest.fn().mockResolvedValue({ deleted: true });
+ const codeDelete = jest.fn().mockResolvedValue({ deleted: true });
getStrategyFunctions.mockImplementation((source) =>
source === FileSources.execute_code
? { deleteFile: codeDelete }
: { deleteFile: primaryDelete },
);
- db.deleteFiles.mockResolvedValue({ deletedCount: 1 });
+ db.deleteFiles.mockImplementation(async (ids) => ({
+ deletedCount: ids.length,
+ deletedFileIds: ids,
+ failedFileIds: [],
+ }));
const req = {
body: {},
- config: {},
+ config: { fileConfig: { retainedFileCleanup: true } },
user: { id: 'user-123', tenantId: 'tenant-a' },
};
const file = {
@@ -2960,7 +3162,11 @@ describe('processDeleteRequest', () => {
expect(primaryDelete).toHaveBeenCalledWith(req, file, undefined);
expect(codeDelete).toHaveBeenCalledWith(req, file);
- expect(db.deleteFiles).toHaveBeenCalledWith(['code-resource-file']);
+ expect(db.deleteFiles).toHaveBeenCalledWith(
+ ['code-resource-file'],
+ undefined,
+ expect.objectContaining({ userId: req.user.id }),
+ );
expect(result).toEqual({ deletedFileIds: ['code-resource-file'], failedFileIds: [] });
});
});
diff --git a/api/server/services/Files/provisionCallback.js b/api/server/services/Files/provisionCallback.js
index 0d3993b35b..5c55f37403 100644
--- a/api/server/services/Files/provisionCallback.js
+++ b/api/server/services/Files/provisionCallback.js
@@ -1,6 +1,7 @@
const { createProvisionFilesCallback: createCallback } = require('@librechat/api');
const { provisionToCodeEnv, provisionToVectorDB } = require('~/server/services/Files/provision');
const db = require('~/models');
+const { getFileStorage } = require('~/server/services/Files/operations');
/* Wiring only: binds this workspace's provisioning service and file model to the
* ON_TOOL_EXECUTE callback implemented in packages/api. */
@@ -15,6 +16,7 @@ function createProvisionFilesCallback({
agentToolContexts,
resolvePrimaryAgentId,
resolveExecutionContext,
+ fileStorage: getFileStorage(),
provisionToCodeEnv,
provisionToVectorDB,
updateFile: db.updateFile,
diff --git a/packages/api/src/agents/__tests__/initialize.test.ts b/packages/api/src/agents/__tests__/initialize.test.ts
index 1d3a2a6660..04582d5fc8 100644
--- a/packages/api/src/agents/__tests__/initialize.test.ts
+++ b/packages/api/src/agents/__tests__/initialize.test.ts
@@ -528,8 +528,20 @@ describe('initializeAgent: allowed providers', () => {
});
describe('initializeAgent: ChatProject context', () => {
+ const filterMock = jest.requireMock('~/files') as {
+ filterFilesByEndpointRuntimeConfig: jest.Mock;
+ };
+ const realFilters = jest.requireActual<typeof import('~/files/filter')>('~/files/filter');
+
beforeEach(() => {
jest.clearAllMocks();
+ filterMock.filterFilesByEndpointRuntimeConfig
+ .mockReset()
+ .mockImplementation(realFilters.filterFilesByEndpointRuntimeConfig);
+ });
+
+ afterEach(() => {
+ filterMock.filterFilesByEndpointRuntimeConfig.mockReset().mockReturnValue([]);
});
const projectFile: TFile = {
@@ -753,6 +765,336 @@ describe('initializeAgent: ChatProject context', () => {
expect(agent).not.toHaveProperty('tool_resources');
expect(db.getFiles).not.toHaveBeenCalled();
});
+ it('queues cleared Project files for the first search in a new conversation', async () => {
+ const { agent, loadTools, db } = createMocks();
+ agent.tools = [Tools.file_search];
+ const repairing = {
+ ...canonicalFile,
+ embedded: false,
+ source: FileSources.s3,
+ metadata: { searchDestination: true },
+ };
+ const runtime = projectRuntime();
+ runtime.chatProjectContext = {
+ ...projectContext,
+ resources: [toCanonicalProjectResource(repairing)],
+ };
+ const result = await initializeAgent(
+ {
+ runtime,
+ agent,
+ loadTools,
+ endpointOption: { endpoint: EModelEndpoint.agents },
+ allowedProviders: new Set([Providers.OPENAI]),
+ useChatProjectContext: true,
+ },
+ db,
+ );
+ expect(result.provisionState?.vectorDBFiles).toEqual([
+ expect.objectContaining({
+ file_id: 'project-file',
+ embedded: false,
+ source: FileSources.s3,
+ metadata: { searchDestination: true },
+ }),
+ ]);
+ expect(result.provisionState?.codeEnvFiles).toEqual([]);
+ expect(result.provisionState?.agentScopedFileIds.size).toBe(0);
+ expect(agent.tools).toEqual([Tools.file_search]);
+ expect(db.getFiles).not.toHaveBeenCalled();
+ expect(db.getProjectFiles).not.toHaveBeenCalled();
+ });
+
+ describe('retained Project repair endpoint policy', () => {
+ const MB = 1024 * 1024;
+ const realResources = jest.requireActual<typeof import('../resources')>('../resources');
+
+ beforeEach(() => {
+ jest.mocked(primeResources).mockImplementationOnce(realResources.primeResources);
+ });
+
+ function repairFile(
+ overrides: Partial<ProjectFileRecord & TFile> = {},
+ ): ProjectFileRecord & TFile {
+ return {
+ ...canonicalFile,
+ embedded: false,
+ source: FileSources.s3,
+ storageKey: 'uploads/project.txt',
+ storageRegion: 'us-east-1',
+ tenantId: 'tenant-1',
+ context: FileContext.message_attachment,
+ bytes: MB / 2,
+ metadata: { searchDestination: true },
+ ...overrides,
+ };
+ }
+
+ async function initializeRepairs({
+ projectFiles = [repairFile()],
+ current = [],
+ deferred = [],
+ persistent = [],
+ fileConfig,
+ filters,
+ }: {
+ projectFiles?: ProjectFileRecord[];
+ current?: TFile[];
+ deferred?: TFile[];
+ persistent?: TFile[];
+ fileConfig?: NonNullable<ServerRequest['config']>['fileConfig'];
+ filters?: FiltersConfig;
+ } = {}) {
+ const { agent, loadTools, db } = createMocks();
+ agent.tools = [Tools.file_search];
+ agent.endpoint = EModelEndpoint.openAI;
+ agent.tool_resources = {
+ [EToolResources.file_search]: { file_ids: persistent.map((file) => file.file_id) },
+ };
+ const runtime = projectRuntime();
+ runtime.user!.tenantId = 'tenant-1';
+ runtime.appConfig.fileConfig = fileConfig;
+ runtime.appConfig.filters = filters;
+ runtime.chatProjectContext = {
+ ...projectContext,
+ file_ids: projectFiles.map((file) => file.file_id),
+ resources: projectFiles.map(toCanonicalProjectResource),
+ };
+ const getFiles = jest.fn().mockResolvedValue(persistent);
+ if (current.length > 0 || deferred.length > 0) {
+ getFiles.mockResolvedValueOnce([...current, ...deferred]);
+ }
+ const getProjectFiles = jest.fn().mockResolvedValue(projectFiles);
+ const getDeferredProvisionFiles = jest.fn().mockResolvedValue(deferred);
+ const result = await initializeAgent(
+ {
+ runtime,
+ agent,
+ loadTools,
+ requestFiles: current.map(({ file_id }) => ({ file_id }) as IMongoFile),
+ conversationId: deferred.length > 0 ? 'repair-conversation' : undefined,
+ allowedProviders: new Set([Providers.OPENAI]),
+ useChatProjectContext: true,
+ fileSearchAvailable: true,
+ },
+ {
+ ...db,
+ getFiles,
+ getProjectFiles,
+ getDeferredProvisionFiles,
+ getConvoFiles: jest.fn().mockResolvedValue(deferred.map((file) => file.file_id)),
+ },
+ );
+ return { result, runtime, getFiles, getProjectFiles, getDeferredProvisionFiles };
+ }
+
+ it.each([
+ { policy: 'disabled files', config: { disabled: true } },
+ { policy: 'MIME allowlist', config: { supportedMimeTypes: ['application/pdf'] } },
+ { policy: 'per-file size', config: { fileSizeLimit: 0.25 } },
+ { policy: 'total size', config: { totalSizeLimit: 0.25 } },
+ ])('does not append or queue a repair rejected by $policy', async ({ config }) => {
+ const { result, getFiles, getProjectFiles } = await initializeRepairs({
+ fileConfig: { endpoints: { [EModelEndpoint.openAI]: config } },
+ });
+ expect(result.tool_resources?.[EToolResources.file_search]?.files ?? []).toEqual([]);
+ expect(result.provisionState?.vectorDBFiles ?? []).toEqual([]);
+ expect(getFiles).not.toHaveBeenCalled();
+ expect(getProjectFiles).not.toHaveBeenCalled();
+ });
+
+ it.each([
+ { routingMimeType: 'text/csv', allowed: 'text/csv', admitted: true },
+ { routingMimeType: 'text/csv', allowed: 'text/plain', admitted: false },
+ { routingMimeType: undefined, allowed: 'text/plain', admitted: true },
+ { routingMimeType: undefined, allowed: 'text/csv', admitted: false },
+ ])(
+ 'screens retained repairs routed as $routingMimeType against $allowed',
+ async ({ routingMimeType, allowed, admitted }) => {
+ const { result, getFiles, getProjectFiles } = await initializeRepairs({
+ projectFiles: [
+ repairFile({
+ type: 'text/plain',
+ metadata: {
+ searchDestination: true,
+ ...(routingMimeType !== undefined ? { routingMimeType } : {}),
+ },
+ }),
+ ],
+ fileConfig: {
+ endpoints: { [EModelEndpoint.openAI]: { supportedMimeTypes: [allowed] } },
+ },
+ });
+ const queued = result.provisionState?.vectorDBFiles ?? [];
+ const attached = result.tool_resources?.[EToolResources.file_search]?.files ?? [];
+ expect(queued).toHaveLength(admitted ? 1 : 0);
+ expect(attached).toHaveLength(admitted ? 1 : 0);
+ if (admitted) {
+ expect(queued[0]).toMatchObject({
+ type: 'text/plain',
+ metadata: {
+ searchDestination: true,
+ ...(routingMimeType !== undefined ? { routingMimeType } : {}),
+ },
+ });
+ }
+ expect(getFiles).not.toHaveBeenCalled();
+ expect(getProjectFiles).not.toHaveBeenCalled();
+ },
+ );
+
+ it.each([undefined, false, true])(
+ 'queues an admitted repair without rereading it when retainedFileCleanup is %p',
+ async (retainedFileCleanup) => {
+ const { result, runtime, getFiles, getProjectFiles } = await initializeRepairs({
+ fileConfig: {
+ ...(retainedFileCleanup === undefined ? {} : { retainedFileCleanup }),
+ endpoints: {
+ [EModelEndpoint.openAI]: {
+ supportedMimeTypes: ['text/plain'],
+ fileSizeLimit: 0.5,
+ totalSizeLimit: 0.5,
+ },
+ },
+ },
+ });
+ const admitted = runtime.chatProjectContext.resources[0];
+ if (admitted.availability !== 'ready') {
+ throw new Error('Expected a ready Project fixture');
+ }
+ expect(result.provisionState?.vectorDBFiles).toEqual([admitted.file]);
+ expect(result.provisionState?.vectorDBFiles[0]).toBe(admitted.file);
+ expect(result.tool_resources?.[EToolResources.file_search]?.files).toEqual([admitted.file]);
+ expect(admitted.file).toMatchObject({
+ embedded: false,
+ user: 'user-1',
+ tenantId: 'tenant-1',
+ source: FileSources.s3,
+ storageKey: 'uploads/project.txt',
+ storageRegion: 'us-east-1',
+ metadata: { searchDestination: true },
+ });
+ expect(result.provisionState?.agentScopedFileIds.size).toBe(0);
+ expect(getFiles).not.toHaveBeenCalled();
+ expect(getProjectFiles).not.toHaveBeenCalled();
+ },
+ );
+
+ it.each([false, true])(
+ 'rejects an over-budget Project repair that overlaps current files (deferred: %s)',
+ async (deferred) => {
+ const oversized = repairFile({ bytes: 2 * MB });
+ const { result } = await initializeRepairs({
+ projectFiles: [oversized],
+ current: [oversized],
+ deferred: deferred ? [oversized] : [],
+ fileConfig: { endpoints: { [EModelEndpoint.openAI]: { totalSizeLimit: 1 } } },
+ });
+ expect(result.provisionState?.vectorDBFiles ?? []).toEqual([]);
+ expect(result.tool_resources?.[EToolResources.file_search]?.files ?? []).toEqual([]);
+ },
+ );
+
+ it.each([false, true])(
+ 'reserves current admission before deferred candidates (overlap first: %s)',
+ async (overlapFirst) => {
+ const current = repairFile({ file_id: 'current', bytes: (MB * 3) / 4 });
+ const other = repairFile({ file_id: 'deferred', bytes: (MB * 3) / 4 });
+ const { result } = await initializeRepairs({
+ projectFiles: [current, other],
+ current: [current],
+ deferred: overlapFirst ? [current, other] : [other, current],
+ fileConfig: { endpoints: { [EModelEndpoint.openAI]: { totalSizeLimit: 1 } } },
+ });
+ expect(result.provisionState?.vectorDBFiles.map((file) => file.file_id)).toEqual([
+ 'current',
+ ]);
+ },
+ );
+
+ it('does not charge duplicate current files twice before admitting other project repairs', async () => {
+ const shared = repairFile({ file_id: 'shared-current', bytes: MB / 2 });
+ const other = repairFile({ file_id: 'other-project', bytes: MB / 2 });
+ const { result } = await initializeRepairs({
+ projectFiles: [shared, other],
+ current: [shared, shared],
+ deferred: [shared, shared],
+ fileConfig: { endpoints: { [EModelEndpoint.openAI]: { totalSizeLimit: 1 } } },
+ });
+ const ids = result.provisionState?.vectorDBFiles.map((file) => file.file_id) ?? [];
+ expect(ids).toEqual(['shared-current', 'other-project']);
+ });
+
+ it('shares one allowance with current, deferred and persistent files, counting overlaps once', async () => {
+ const shared = repairFile({ file_id: 'shared', bytes: MB / 4 });
+ const currentOnly = repairFile({ file_id: 'current', bytes: MB / 8 });
+ const deferredOnly = repairFile({ file_id: 'deferred', bytes: MB / 8 });
+ const persistentOnly = repairFile({ file_id: 'persistent', bytes: MB / 4 });
+ const fits = repairFile({ file_id: 'fits', bytes: MB / 4 });
+ const overflow = repairFile({ file_id: 'overflow', bytes: MB / 8 });
+ const { result, getFiles, getProjectFiles, getDeferredProvisionFiles } =
+ await initializeRepairs({
+ projectFiles: [shared, persistentOnly, fits, overflow],
+ current: [shared, currentOnly],
+ deferred: [shared, deferredOnly],
+ persistent: [shared, persistentOnly],
+ fileConfig: { endpoints: { [EModelEndpoint.openAI]: { totalSizeLimit: 1 } } },
+ });
+ const queuedIds = result.provisionState?.vectorDBFiles.map((file) => file.file_id) ?? [];
+ expect(queuedIds).toEqual(
+ expect.arrayContaining(['shared', 'current', 'deferred', 'persistent', 'fits']),
+ );
+ expect(queuedIds).not.toContain('overflow');
+ expect(new Set(queuedIds).size).toBe(queuedIds.length);
+ expect(result.tool_resources?.[EToolResources.file_search]?.files).toEqual(
+ expect.arrayContaining([expect.objectContaining({ file_id: 'fits' })]),
+ );
+ expect(result.tool_resources?.[EToolResources.file_search]?.files).not.toEqual(
+ expect.arrayContaining([expect.objectContaining({ file_id: 'overflow' })]),
+ );
+ expect(filterMock.filterFilesByEndpointRuntimeConfig).toHaveBeenLastCalledWith(
+ expect.anything(),
+ expect.objectContaining({
+ consumedBytes: (MB * 3) / 4,
+ files: [
+ expect.objectContaining({ file_id: 'fits' }),
+ expect.objectContaining({ file_id: 'overflow' }),
+ ],
+ }),
+ );
+ expect(getFiles).toHaveBeenCalledTimes(2);
+ expect(getProjectFiles).not.toHaveBeenCalled();
+ expect(getDeferredProvisionFiles).toHaveBeenCalledTimes(1);
+ });
+
+ it.each(['endpoint', 'content'])(
+ 'does not charge persistent files rejected by %s policy',
+ async (policy) => {
+ const persistent = repairFile({
+ file_id: 'rejected-persistent',
+ bytes: (MB * 3) / 4,
+ ...(policy === 'endpoint' ? { type: 'application/pdf' } : { text: 'BLOCKED' }),
+ });
+ const { result } = await initializeRepairs({
+ persistent: [persistent],
+ filters: policy === 'content' ? projectFilePolicy : undefined,
+ fileConfig: {
+ endpoints: {
+ [EModelEndpoint.openAI]: { supportedMimeTypes: ['text/plain'], totalSizeLimit: 1 },
+ },
+ },
+ });
+ expect(result.provisionState?.vectorDBFiles.map((file) => file.file_id)).toEqual([
+ projectFile.file_id,
+ ]);
+ expect(filterMock.filterFilesByEndpointRuntimeConfig).toHaveBeenLastCalledWith(
+ expect.anything(),
+ expect.objectContaining({ consumedBytes: 0 }),
+ );
+ },
+ );
+ });
+
it('adds Project files for file_search supplied by a manual Skill', async () => {
const { agent, req, loadTools, db } = createMocks();
const { Types } = await import('mongoose');
@@ -2953,6 +3295,12 @@ describe('initializeAgent — skill `allowed-tools` union (Phase 6)', () => {
expect(definedNames).not.toContain('mcp__broken__tool');
});
it('removes skill-added Project resources before retrying without denied tools', async () => {
+ const { filterFilesByEndpointRuntimeConfig } = jest.requireMock('~/files') as {
+ filterFilesByEndpointRuntimeConfig: jest.Mock;
+ };
+ filterFilesByEndpointRuntimeConfig.mockImplementationOnce(
+ (_config: unknown, { files }: { files: TFile[] }) => files,
+ );
const { agent, req, loadTools, db } = createMocks();
const { Types } = await import('mongoose');
const skillId = new Types.ObjectId();
@@ -4510,10 +4858,10 @@ describe('initializeAgent — code-generated file thread filter (regression)', (
const deferredOnly = { file_id: 'deferred', filename: 'b.csv', bytes: 100 };
const { agent, req, res, loadTools, db } = setupExecuteCodeAgent();
- /* Delivery keeps the shared file; the deferred pass keeps both. */
+ /* Delivery has admitted the shared file; deferred screening sees only new ids. */
filterFilesByEndpointRuntimeConfig
.mockReturnValueOnce([shared])
- .mockReturnValueOnce([shared, deferredOnly]);
+ .mockReturnValueOnce([deferredOnly]);
const getDeferredProvisionFiles = jest.fn().mockResolvedValue([shared, deferredOnly]);
const getConvoFiles = jest.fn().mockResolvedValue(['shared', 'deferred']);
const getToolFilesByIds = jest.fn().mockResolvedValue([shared]);
@@ -4542,9 +4890,12 @@ describe('initializeAgent — code-generated file thread filter (regression)', (
const chargedToDeferred = filterFilesByEndpointRuntimeConfig.mock.calls
.map(([, params]) => (params as { consumedBytes?: number }).consumedBytes)
.find((bytes) => bytes !== undefined);
- /* The only delivered file is also the shared one, and the deferred pass charges its
- * own list as it walks it, so nothing is carried in. */
- expect(chargedToDeferred).toBe(0);
+ expect(chargedToDeferred).toBe(400);
+ expect(filterFilesByEndpointRuntimeConfig).toHaveBeenNthCalledWith(
+ 2,
+ expect.anything(),
+ expect.objectContaining({ files: [deferredOnly], consumedBytes: 400 }),
+ );
/* The persistent screening runs inside the callback, and must see the two unique
* files rather than three. */
diff --git a/packages/api/src/agents/__tests__/run-codeTools.test.ts b/packages/api/src/agents/__tests__/run-codeTools.test.ts
index 286735db7d..e6da4f20cd 100644
--- a/packages/api/src/agents/__tests__/run-codeTools.test.ts
+++ b/packages/api/src/agents/__tests__/run-codeTools.test.ts
@@ -1,12 +1,14 @@
import { FileContext } from 'librechat-data-provider';
import type { AgentInputs, SubagentTaskConfig, SubagentResolveContext } from '@librechat/agents';
+import type { FileProvisionCommit } from '@librechat/data-schemas';
import type { TFile } from 'librechat-data-provider';
+import type { ProvisionCallbackDeps, ProvisionToolContext } from '~/files/provision/callback';
import type { HostSubagentTaskConfig } from '~/agents/subagentDelivery';
-import type { ProvisionToolContext } from '~/files/provision/callback';
import type { ServerRequest } from '~/types';
import { createProvisionFilesCallback } from '~/files/provision/callback';
import { SUBAGENT_COMPLETION_DELIVERY } from '~/agents/subagentDelivery';
import { mergeCodeFilesIntoContext } from '~/agents/codeFilesSession';
+import { createFileProvisionRecovery } from '../../files/operations';
import { CHECK_BACKGROUND_TASK_NAME } from '~/agents/background';
import { createRun } from '~/agents/run';
@@ -441,14 +443,43 @@ describe('createRun code-tool eager/session wiring', () => {
};
},
);
+ const updateFile = jest.fn(
+ async (_update: Parameters<ProvisionCallbackDeps['updateFile']>[0]): Promise<unknown> =>
+ undefined,
+ );
+ const updateCodeEnvRef = jest.fn(
+ async (
+ _update: Parameters<ProvisionCallbackDeps['updateCodeEnvRef']>[0],
+ ): Promise<unknown> => undefined,
+ );
+ const addEmbeddedEntity = jest.fn(
+ async (
+ _update: Parameters<ProvisionCallbackDeps['addEmbeddedEntity']>[0],
+ ): Promise<unknown> => undefined,
+ );
const provision = createProvisionFilesCallback({
req: { user: { id: 'user-1' } } as ServerRequest,
agentToolContexts: contexts,
+ fileStorage: {
+ recovery: createFileProvisionRecovery(),
+ acquireFileProvision: jest.fn(async () => true),
+ releaseFileProvision: jest.fn(async () => true),
+ commitFileProvision: jest.fn(
+ async ({ claim, code, entityId, embedded }: FileProvisionCommit) => {
+ let result: unknown;
+ if (code) result = await updateCodeEnvRef({ file_id: claim.file_id, ...code });
+ else if (entityId != null)
+ result = await addEmbeddedEntity({ file_id: claim.file_id, entityId });
+ else result = await updateFile({ file_id: claim.file_id, embedded });
+ return result !== null && result !== false;
+ },
+ ),
+ },
provisionToCodeEnv,
provisionToVectorDB: jest.fn(),
- updateFile: jest.fn(),
- updateCodeEnvRef: jest.fn(),
- addEmbeddedEntity: jest.fn(),
+ updateFile,
+ updateCodeEnvRef,
+ addEmbeddedEntity,
});
for (const child of children) {
contexts.set(child.id, child);
diff --git a/packages/api/src/agents/attachments.test.ts b/packages/api/src/agents/attachments.test.ts
index e4a7678143..52bb69ce12 100644
--- a/packages/api/src/agents/attachments.test.ts
+++ b/packages/api/src/agents/attachments.test.ts
@@ -34,6 +34,21 @@ const makeTextFile = (file_id: string, filename: string, text: string): IMongoFi
source: FileSources.text,
}) as IMongoFile;
+describe('retained search ownership', () => {
+ it('keeps confirmed-deleted code destinations tool-owned without a live pointer', () => {
+ const file = { embedded: false, metadata: { codeDestination: true } } as IMongoFile;
+ expect(isToolOwnedAttachment(file)).toBe(true);
+ expect(isModelBoundAttachmentFile(file)).toBe(false);
+ });
+
+ it('keeps markerless search files tool-owned while vectors await repair', () => {
+ const file = { embedded: false, metadata: { searchDestination: true } } as IMongoFile;
+ expect(isToolOwnedAttachment(file)).toBe(true);
+ expect(isModelBoundAttachmentFile(file)).toBe(false);
+ expect(isToolOwnedAttachment({ embedded: false } as IMongoFile)).toBe(false);
+ });
+});
+
describe('agent attachment helpers', () => {
it('excludes tool-only files while retaining extracted text files', () => {
expect(isModelBoundAttachmentFile({ file_id: 'rag', embedded: true } as IMongoFile)).toBe(
diff --git a/packages/api/src/agents/attachments.ts b/packages/api/src/agents/attachments.ts
index 04885b05c2..4745ae0964 100644
--- a/packages/api/src/agents/attachments.ts
+++ b/packages/api/src/agents/attachments.ts
@@ -45,6 +45,8 @@ export function isToolOwnedAttachment(file: AttachmentTelemetryFile): boolean {
return (
file.context === FileContext.execute_code ||
file.embedded === true ||
+ metadata?.searchDestination === true ||
+ metadata?.codeDestination === true ||
metadata?.codeEnvRef != null ||
metadata?.codeEnvRefs != null ||
metadata?.fileIdentifier != null
diff --git a/packages/api/src/agents/files/budget.spec.ts b/packages/api/src/agents/files/budget.spec.ts
new file mode 100644
index 0000000000..9ba46167e3
--- /dev/null
+++ b/packages/api/src/agents/files/budget.spec.ts
@@ -0,0 +1,41 @@
+import { admitCurrentFileBudget } from './budget';
+
+it('rejects an overlapping current file before priming can reuse its admission', () => {
+ const file = { file_id: 'retained-project', bytes: 2 * 1024 * 1024 };
+ expect(admitCurrentFileBudget([file], 1024 * 1024)).toEqual([]);
+});
+
+it('charges duplicate current files only once under a shared total', () => {
+ const first = { file_id: 'current', bytes: 512 };
+ const other = { file_id: 'other', bytes: 512 };
+ expect(admitCurrentFileBudget([first, first, other], 1024)).toEqual([first, other]);
+});
+
+it('keeps later files that fit after rejecting an over-budget candidate', () => {
+ const files = [
+ { file_id: 'first', bytes: 512 },
+ { file_id: 'oversized', bytes: 2048 },
+ { file_id: 'last', bytes: 512 },
+ ];
+ expect(admitCurrentFileBudget(files, 1024)).toEqual([files[0], files[2]]);
+});
+
+it.each([undefined, 0, -1])('retains unique inputs without a positive limit: %s', (limit) => {
+ const file = { file_id: 'first', bytes: 2048 };
+ expect(admitCurrentFileBudget([file, file], limit)).toEqual([file]);
+});
+
+it('does not collapse files without identities or mutate the original snapshots', () => {
+ const first = Object.freeze({ bytes: 512 });
+ const other = Object.freeze({ bytes: 512 });
+ const files = [first, other];
+ const result = admitCurrentFileBudget(files, 1024);
+ expect(result).toEqual(files);
+ expect(result).not.toBe(files);
+ expect(result[0]).toBe(first);
+});
+
+it('tolerates absent byte counts on an otherwise admitted snapshot', () => {
+ const file = { file_id: 'legacy' };
+ expect(admitCurrentFileBudget([file], 1)).toEqual([file]);
+});
diff --git a/packages/api/src/agents/files/budget.ts b/packages/api/src/agents/files/budget.ts
new file mode 100644
index 0000000000..ed1b4903a5
--- /dev/null
+++ b/packages/api/src/agents/files/budget.ts
@@ -0,0 +1,22 @@
+/** Already MIME/size-screened files spend the shared allowance once before priming. */
+export function admitCurrentFileBudget<T extends { file_id?: string; bytes?: number }>(
+ files: T[],
+ totalSizeLimit?: number,
+): T[] {
+ const admitted: T[] = [];
+ const seen = new Set<string>();
+ let consumed = 0;
+ for (const file of files) {
+ if (file.file_id != null) {
+ if (seen.has(file.file_id)) continue;
+ seen.add(file.file_id);
+ }
+ const bytes = file.bytes ?? 0;
+ if (totalSizeLimit != null && totalSizeLimit > 0 && consumed + bytes > totalSizeLimit) {
+ continue;
+ }
+ admitted.push(file);
+ consumed += bytes;
+ }
+ return admitted;
+}
diff --git a/packages/api/src/agents/files/host.spec.ts b/packages/api/src/agents/files/host.spec.ts
index 51cea7b292..51bffaa865 100644
--- a/packages/api/src/agents/files/host.spec.ts
+++ b/packages/api/src/agents/files/host.spec.ts
@@ -7,12 +7,14 @@ import {
configSchema,
} from 'librechat-data-provider';
import type { Agent, CodeEnvRef, TFile } from 'librechat-data-provider';
+import type { FileProvisionCommit } from '@librechat/data-schemas';
import type { SubagentExecutionContext } from '@librechat/agents';
import type { ProvisionCallbackDeps } from '~/files/provision/callback';
import type { RunFileSessionDeps } from './session';
import type { RunFileToolContext } from './host';
import type { ServerRequest } from '~/types';
import { createRunFileHost, resolveRunFileCodeExecutionContext } from './host';
+import { createFileProvisionRecovery } from '../../files/operations';
import * as sessionModule from './session';
jest.mock('@librechat/data-schemas', () => ({
@@ -158,7 +160,9 @@ function harness(
id: entity_id ?? 'user',
file_id: `remote-${candidate.file_id}`,
storage_session_id: `storage-${entity_id ?? 'user'}`,
- sandboxFilename,
+ executionProfile: route?.executionProfile ?? 'default',
+ executionRouteKey: routeKey,
+ sandboxFilename: sandboxFilename ?? candidate.filename,
};
return {
referenceSet: { codeEnvRefs: { [routeKey]: ref } },
@@ -176,8 +180,18 @@ function harness(
embedded: true,
fileUpdate: { file_id: candidate.file_id, embedded: true },
}));
- const updateCodeEnvRef = jest.fn(async () => undefined);
- const addEmbeddedEntity = jest.fn(async () => undefined);
+ const updateCodeEnvRef = jest.fn(
+ async (_update: Parameters<ProvisionCallbackDeps['updateCodeEnvRef']>[0]): Promise<unknown> =>
+ undefined,
+ );
+ const addEmbeddedEntity = jest.fn(
+ async (_update: Parameters<ProvisionCallbackDeps['addEmbeddedEntity']>[0]): Promise<unknown> =>
+ undefined,
+ );
+ const updateFile = jest.fn(
+ async (_update: Parameters<ProvisionCallbackDeps['updateFile']>[0]): Promise<unknown> =>
+ undefined,
+ );
const emit = jest.fn(async () => undefined);
const encodeMessages = jest.fn(async () => []);
const spy = jest.spyOn(sessionModule, 'createRunFileSession');
@@ -200,10 +214,25 @@ function harness(
validateMessages: jest.fn(),
encodeMessages,
provisioning: {
+ fileStorage: {
+ recovery: createFileProvisionRecovery(),
+ acquireFileProvision: jest.fn(async () => true),
+ releaseFileProvision: jest.fn(async () => true),
+ commitFileProvision: jest.fn(
+ async ({ claim, code, entityId, embedded }: FileProvisionCommit) => {
+ let result: unknown;
+ if (code) result = await updateCodeEnvRef({ file_id: claim.file_id, ...code });
+ else if (entityId != null)
+ result = await addEmbeddedEntity({ file_id: claim.file_id, entityId });
+ else result = await updateFile({ file_id: claim.file_id, embedded });
+ return result !== null && result !== false;
+ },
+ ),
+ },
provisionToCodeEnv,
provisionToVectorDB,
updateCodeEnvRef,
- updateFile: jest.fn(async () => undefined),
+ updateFile,
addEmbeddedEntity,
hasEntityEmbeddings: options.hasEntityEmbeddings,
adoptLegacySearchEmbeddings: options.adoptLegacySearchEmbeddings,
@@ -678,7 +707,16 @@ describe('run file execution host', () => {
const h = harness({ setup: [] });
const first = identity('first');
const second = identity('second');
- await Promise.all([h.prepare(first), h.prepare(second)]);
+ const same = identity('same');
+ await Promise.all([h.prepare(first), h.prepare(second), h.prepare(same)]);
+ const secondContext = h.host.getContext('worker', second)!;
+ await h.deps.prepareAgent({
+ actor: h.host.session.actorFor('worker', second),
+ files: [file('independent')],
+ sessionKey: secondContext.codeExecutionContext!.codeSessionKey,
+ revision: 100,
+ signal: h.signal,
+ });
const entered = deferred<void>();
const finish = deferred<void>();
h.provisionToCodeEnv.mockImplementationOnce(async (params) => {
@@ -701,14 +739,44 @@ describe('run file execution host', () => {
.then(() => {
refreshed = true;
});
- await h.host.provision([Constants.EXECUTE_CODE], 'worker', h.signal, second);
- expect(refreshed).toBe(false);
- finish.resolve();
- await Promise.all([upload, refresh]);
+ let sharedSettled = false;
+ const shared = h.host
+ .provisionPrepared([Constants.EXECUTE_CODE], 'worker', h.signal, same)
+ .then((result) => {
+ sharedSettled = true;
+ return result;
+ });
+ try {
+ const independent = await h.host.provisionPrepared(
+ [Constants.EXECUTE_CODE],
+ 'worker',
+ h.signal,
+ second,
+ );
+ expect(independent?.map((entry) => entry.id)).toEqual(['remote-independent']);
+ expect(refreshed).toBe(false);
+ expect(sharedSettled).toBe(false);
+ } finally {
+ finish.resolve();
+ await Promise.all([upload, refresh, shared]);
+ }
+ expect((await shared)?.map((entry) => entry.id)).toEqual(['remote-input']);
+ expect(
+ h.provisionToCodeEnv.mock.calls.filter(([params]) => params.file.file_id === 'input'),
+ ).toHaveLength(1);
expect(h.host.getContext('worker', first)).toBe(before);
expect(before.tool_resources?.execute_code?.files?.map((entry) => entry.file_id)).toEqual([
'input',
]);
+ const provisionedInput = before.tool_resources?.execute_code?.files?.find(
+ (entry) => entry.file_id === 'input',
+ );
+ expect(provisionedInput?.metadata?.codeEnvRefs?.['managed-worker']).toMatchObject({
+ file_id: 'remote-input',
+ executionProfile: 'stateful',
+ executionRouteKey: 'managed-worker',
+ sandboxFilename: 'input.csv',
+ });
expect(before.provisionState?.codeEnvFiles.map((entry) => entry.file_id)).toEqual(['new']);
});
diff --git a/packages/api/src/agents/files/host.ts b/packages/api/src/agents/files/host.ts
index bcd07a4eb1..5cd8f2aad9 100644
--- a/packages/api/src/agents/files/host.ts
+++ b/packages/api/src/agents/files/host.ts
@@ -107,6 +107,7 @@ export function createRunFileHost<TContext extends RunFileToolContext>({
audit?: RunFileSessionDeps['audit'];
provisioning: Pick<
ProvisionCallbackDeps,
+ | 'fileStorage'
| 'provisionToCodeEnv'
| 'provisionToVectorDB'
| 'updateFile'
diff --git a/packages/api/src/agents/initialize.ts b/packages/api/src/agents/initialize.ts
index 727acd32d6..ccb4f54ab1 100644
--- a/packages/api/src/agents/initialize.ts
+++ b/packages/api/src/agents/initialize.ts
@@ -6,6 +6,7 @@ import {
ErrorTypes,
EModelEndpoint,
EToolResources,
+ canToolResourceConsume,
paramEndpoints,
isAgentsEndpoint,
AgentCapabilities,
@@ -143,6 +144,7 @@ import { hasActiveFilePolicy } from '../protection/files';
import { resolveAgentCodeExecution } from '~/code/agent';
import { hasActiveFileFieldPolicy } from '~/protection';
import { applyBackgroundToolCalls } from './background';
+import { admitCurrentFileBudget } from './files/budget';
import { applyTurnDelivery } from './files/delivery';
import { generateArtifactsPrompt } from '~/prompts';
import { getAgentInferencePolicy } from './input';
@@ -179,7 +181,12 @@ export function partitionCommittedFiles<T extends { file_id?: string }>(
);
const alreadyCommitted: T[] = [];
const pending: T[] = [];
+ const seen = new Set<string>();
for (const file of files) {
+ if (file.file_id != null) {
+ if (seen.has(file.file_id)) continue;
+ seen.add(file.file_id);
+ }
if (file.file_id != null && ids.has(file.file_id)) {
alreadyCommitted.push(file);
continue;
@@ -309,32 +316,59 @@ function addProjectFilesToFileSearch(
effectiveToolNames: readonly string[],
appConfig: AppConfig | undefined,
fileSearchAvailable?: boolean,
-): AgentToolResources | undefined {
+ provisionState?: ProvisionState,
+): { resources: AgentToolResources | undefined; provisionState?: ProvisionState } {
if (files == null || files.length === 0) {
- return resources;
+ return { resources, provisionState };
}
const capabilities = appConfig?.endpoints?.[EModelEndpoint.agents]?.capabilities ?? [];
if (!capabilities.includes(AgentCapabilities.file_search)) {
- return resources;
+ return { resources, provisionState };
}
if (fileSearchAvailable === false) {
- return resources;
+ return { resources, provisionState };
}
if (!effectiveToolNames.includes(Tools.file_search)) {
- return resources;
+ return { resources, provisionState };
}
const current = resources?.[EToolResources.file_search] ?? {};
const currentFiles = current.files ?? [];
const seen = new Set(currentFiles.map((file) => file.file_id));
- const projectFiles = files.filter((file) => !seen.has(file.file_id));
+ const projectFiles: TFile[] = [];
+ const queuedIds = new Set(provisionState?.vectorDBFiles.map((file) => file.file_id));
+ for (const file of files) {
+ if (!seen.has(file.file_id)) {
+ projectFiles.push(file);
+ seen.add(file.file_id);
+ }
+ if (
+ file.embedded === true ||
+ file.metadata?.searchDestination !== true ||
+ !canToolResourceConsume(EToolResources.file_search, file.type) ||
+ queuedIds.has(file.file_id)
+ ) {
+ continue;
+ }
+ provisionState ??= {
+ codeEnvFiles: [],
+ vectorDBFiles: [],
+ aliveFileIds: new Set(),
+ agentScopedFileIds: new Set(),
+ };
+ provisionState.vectorDBFiles.push(file);
+ queuedIds.add(file.file_id);
+ }
if (projectFiles.length === 0) {
- return resources;
+ return { resources, provisionState };
}
return {
- ...(resources ?? {}),
- [EToolResources.file_search]: {
- ...current,
- files: currentFiles.concat(projectFiles),
+ provisionState,
+ resources: {
+ ...(resources ?? {}),
+ [EToolResources.file_search]: {
+ ...current,
+ files: currentFiles.concat(projectFiles),
+ },
},
};
}
@@ -1923,36 +1957,26 @@ export async function initializeAgent(
endpointType,
});
+ currentFiles = admitCurrentFileBudget(
+ currentFiles,
+ deliveryRouting.endpointConfig.totalSizeLimit,
+ );
+
/* The same endpoint configuration governs both paths. A file this endpoint refuses
* by size, MIME type, or a files-disabled setting must not reach the Code API or
* RAG through provisioning just because it left the delivery set. */
if (deferredProvisionFiles.length > 0) {
- /* One request, one total-size allowance. Filtering each set from zero would let a
- * delivery attachment and a provisioning candidate that each fit alone exceed the
- * limit together once withDeferredCandidates merges them.
- *
- * A file can appear in both sets, an embedded attachment still missing the active
- * code route being the case in point, and the merge deduplicates afterwards. Charging
- * it twice would spend an allowance the request never uses and drop a different
- * candidate that fits, so the shared ones are counted once. */
- const deferredFileIds = new Set(
- deferredProvisionFiles
- .map((file) => file.file_id)
- .filter((fileId): fileId is string => fileId != null),
+ const { committed, pending } = partitionCommittedFiles(deferredProvisionFiles, currentFiles);
+ /* Current admission is final. Charge only new deferred identities against its
+ * remainder, regardless of their order relative to overlapping current files. */
+ deferredProvisionFiles = committed.concat(
+ filterFilesByEndpointRuntimeConfig(appConfig, {
+ files: pending,
+ endpoint: agent.endpoint ?? '',
+ endpointType,
+ consumedBytes: sumUniqueBytes(currentFiles),
+ }) as IMongoFile[],
);
- deferredProvisionFiles = filterFilesByEndpointRuntimeConfig(appConfig, {
- files: deferredProvisionFiles,
- endpoint: agent.endpoint ?? '',
- endpointType,
- /* The deferred pass charges its own list as it walks it, so a file in both sets
- * is counted there. Only what delivery spends on files the deferred pass will
- * not see is carried in. */
- consumedBytes: sumUniqueBytes(
- (currentFiles ?? []).filter(
- (file) => file.file_id == null || !deferredFileIds.has(file.file_id),
- ),
- ),
- }) as IMongoFile[];
}
}
@@ -2047,12 +2071,13 @@ export async function initializeAgent(
runtimeToolResources = { ...runtimeToolResources };
delete runtimeToolResources[EToolResources.execute_code];
}
+ let admittedPersistentFiles: TFile[] = [];
const {
attachments: primedAttachments,
requestAttachments: primedRequestAttachments,
agentContextAttachments: primedAgentContextAttachments,
tool_resources,
- provisionState,
+ provisionState: primedProvisionState,
warnings: provisionWarnings,
} = await primeResources({
req: params.req,
@@ -2094,7 +2119,7 @@ export async function initializeAgent(
/* Dropped rather than fatal, matching the deferred candidates: these were not
* attached by this request, so refusing the conversation over a historical record
* would be harsher than leaving it out. */
- return committed.concat(withinPolicy).filter((file) => {
+ admittedPersistentFiles = committed.concat(withinPolicy).filter((file) => {
try {
assertModelBoundContent({
filters: appConfig?.filters,
@@ -2109,15 +2134,37 @@ export async function initializeAgent(
return false;
}
});
+ return admittedPersistentFiles;
},
});
- runtimeToolResources = addProjectFilesToFileSearch(
+ if (projectRuntimeFiles.length > 0) {
+ /* Project repairs join after priming, but spend the same allowance as the files
+ * already admitted above. Reuse their snapshots without charging overlaps twice. */
+ const committedFiles = [
+ ...(currentFiles ?? []),
+ ...deferredProvisionFiles,
+ ...admittedPersistentFiles,
+ ];
+ const { committed, pending } = partitionCommittedFiles(projectRuntimeFiles, committedFiles);
+ projectRuntimeFiles = committed.concat(
+ filterFilesByEndpointRuntimeConfig(appConfig, {
+ files: pending,
+ endpoint: agent.endpoint ?? '',
+ endpointType: endpointFileType,
+ consumedBytes: sumUniqueBytes(committedFiles),
+ }),
+ );
+ }
+ const projectResources = addProjectFilesToFileSearch(
tool_resources,
projectRuntimeFiles,
effectiveToolNames,
appConfig,
params.fileSearchAvailable,
+ primedProvisionState,
);
+ runtimeToolResources = projectResources.resources;
+ const provisionState = projectResources.provisionState;
const dropFileSearchResources = (): void => {
if (runtimeToolResources?.[EToolResources.file_search] == null) {
return;
diff --git a/packages/api/src/agents/resources.test.ts b/packages/api/src/agents/resources.test.ts
index 46f43f2b63..0930501edb 100644
--- a/packages/api/src/agents/resources.test.ts
+++ b/packages/api/src/agents/resources.test.ts
@@ -10,6 +10,8 @@ import type { TAgentsEndpoint, TFile } from 'librechat-data-provider';
import type { IUser, AppConfig } from '@librechat/data-schemas';
import type { Request as ServerRequest } from 'express';
import type { TGetFiles, TFilterFilesByAgentAccess } from './resources';
+import { createDeleteFileWithSecondaryStorage } from '../files/storage';
+import { createFileProvisionRecovery } from '../files/operations';
import { primeResources } from './resources';
// Mock logger
@@ -139,6 +141,66 @@ describe('primeResources', () => {
]);
});
+ it.each(['primary', 'code'])('re-provisions saved files on %s failure', async (failure) => {
+ const file = sharedContextFile(['agent-a']);
+ file.metadata = {
+ ...file.metadata,
+ codeEnvRef: {
+ kind: 'agent',
+ id: 'agent-a',
+ storage_session_id: 'session',
+ file_id: 'code-copy',
+ },
+ };
+ const providerError = new Error('provider unavailable');
+ const primary = jest.fn().mockResolvedValue(undefined);
+ const code = jest.fn().mockResolvedValue({ deleted: true });
+ (failure === 'primary' ? primary : code).mockRejectedValue(providerError);
+ const clearFileEmbeddings = jest.fn().mockResolvedValue(true);
+ const remove = createDeleteFileWithSecondaryStorage({
+ source: FileSources.local,
+ deleteFile: primary,
+ getDeleteFile: (source) =>
+ source === FileSources.vectordb ? async () => ({ deleted: true }) : code,
+ clearFileEmbeddings,
+ isFileStorageClean: jest.fn().mockResolvedValue(true),
+ fileStorage: {
+ recovery: createFileProvisionRecovery(),
+ acquireFileProvision: jest.fn().mockResolvedValue(true),
+ releaseFileProvision: jest.fn().mockResolvedValue(true),
+ commitFileProvision: jest.fn().mockResolvedValue(true),
+ acquireFileDeletion: jest.fn().mockResolvedValue({ resumed: false }),
+ releaseFileDeletion: jest.fn().mockResolvedValue(true),
+ },
+ logger: { warn: jest.fn() },
+ });
+ await expect(
+ remove(
+ { user: { id: 'user1' }, config: { fileConfig: { retainedFileCleanup: true } } },
+ file,
+ ),
+ ).rejects.toBe(providerError);
+ expect(file.embedded).toBe(false);
+ expect(file.metadata?.embeddedEntities).toBeUndefined();
+ expect(file.metadata?.codeEnvRef?.file_id).toBe('code-copy');
+ mockGetFiles.mockResolvedValue([file]);
+ const result = await primeResources({
+ req: mockReq,
+ appConfig: mockAppConfig,
+ getFiles: mockGetFiles,
+ filterFiles: mockFilterFiles,
+ requestFileSet,
+ attachments: undefined,
+ tool_resources: { [EToolResources.file_search]: { file_ids: [file.file_id] } },
+ agentId: 'agent-a',
+ enabledToolResources: new Set([EToolResources.file_search]),
+ });
+ expect(result.provisionState?.vectorDBFiles.map((candidate) => candidate.file_id)).toContain(
+ file.file_id,
+ );
+ expect(result.tool_resources?.file_search?.file_ids).toContain(file.file_id);
+ });
+
it('does not re-embed for the agent that already provisioned it', async () => {
const result = await primeFor('agent-a', sharedContextFile(['agent-a']));
@@ -146,6 +208,116 @@ describe('primeResources', () => {
});
});
+ describe('search destination after confirmed vector cleanup', () => {
+ const chatFile = (destinationChosen?: boolean): TFile => ({
+ user: 'user1',
+ file_id: 'chat-search',
+ filename: 'notes.pdf',
+ filepath: '/uploads/notes.pdf',
+ object: 'file',
+ type: 'application/pdf',
+ bytes: 1024,
+ usage: 0,
+ source: FileSources.local,
+ context: FileContext.message_attachment,
+ llmDeliveryPath: 'none',
+ embedded: true,
+ metadata: {
+ ...(destinationChosen !== undefined ? { destinationChosen } : {}),
+ embeddedEntities: ['user1'],
+ },
+ });
+
+ const primeChat = (file: TFile, enabledToolResources: Set<EToolResources>) =>
+ primeResources({
+ req: mockReq,
+ appConfig: mockAppConfig,
+ getFiles: mockGetFiles,
+ filterFiles: mockFilterFiles,
+ requestFileSet,
+ attachments: Promise.resolve([file]),
+ tool_resources: {},
+ agentId: 'agent-a',
+ enabledToolResources,
+ });
+
+ it.each([true, undefined, false])(
+ 'requeues search after primary failure with chooser marker %s',
+ async (destinationChosen) => {
+ const file = chatFile(destinationChosen);
+ const failure = new Error('primary unavailable');
+ const remove = createDeleteFileWithSecondaryStorage({
+ source: FileSources.local,
+ deleteFile: async () => {
+ throw failure;
+ },
+ getDeleteFile: () => async () => ({ deleted: true }),
+ clearFileEmbeddings: jest.fn().mockResolvedValue(true),
+ isFileStorageClean: jest.fn().mockResolvedValue(true),
+ fileStorage: {
+ recovery: createFileProvisionRecovery(),
+ acquireFileProvision: jest.fn().mockResolvedValue(true),
+ releaseFileProvision: jest.fn().mockResolvedValue(true),
+ commitFileProvision: jest.fn().mockResolvedValue(true),
+ acquireFileDeletion: jest.fn().mockResolvedValue({ resumed: false }),
+ releaseFileDeletion: jest.fn().mockResolvedValue(true),
+ },
+ logger: { warn: jest.fn() },
+ });
+ await expect(
+ remove(
+ { user: { id: 'user1' }, config: { fileConfig: { retainedFileCleanup: true } } },
+ file,
+ ),
+ ).rejects.toBe(failure);
+ expect(file.embedded).toBe(false);
+ expect(file.metadata?.embeddedEntities).toBeUndefined();
+ expect(file.metadata?.searchDestination).toBe(true);
+ expect(file.llmDeliveryPath).toBe('none');
+ const enabled = new Set([EToolResources.file_search, EToolResources.execute_code]);
+ const result = await primeChat(file, enabled);
+ expect(result.provisionState?.vectorDBFiles.map((candidate) => candidate.file_id)).toEqual([
+ file.file_id,
+ ]);
+ expect(result.provisionState?.agentScopedFileIds.has(file.file_id)).toBe(false);
+ expect(file.embedded).toBe(false);
+ if (destinationChosen !== false) {
+ expect(result.provisionState?.codeEnvFiles).toEqual([]);
+ }
+ expect(mockGetFiles).not.toHaveBeenCalled();
+ },
+ );
+
+ it.each([true, undefined])(
+ 'does not provision a declined search destination (%s)',
+ async (choice) => {
+ const file = chatFile(choice);
+ file.embedded = false;
+ file.metadata = {
+ destinationChosen: choice,
+ codeEnvRef: {
+ kind: 'user',
+ id: 'user1',
+ storage_session_id: 'session',
+ file_id: 'code-copy',
+ },
+ };
+ const result = await primeChat(file, new Set([EToolResources.file_search]));
+ expect(result.provisionState?.vectorDBFiles ?? []).toEqual([]);
+ expect(file.metadata.searchDestination).toBeUndefined();
+ },
+ );
+
+ it('does not enable a tool or provision search when File Search is disabled', async () => {
+ const file = chatFile(true);
+ file.embedded = false;
+ file.metadata = { destinationChosen: true, searchDestination: true };
+ const result = await primeChat(file, new Set([EToolResources.execute_code]));
+ expect(result.provisionState).toBeUndefined();
+ expect(result.tool_resources?.file_search).toBeUndefined();
+ });
+ });
+
describe('when policy screening rejects a persistent context file', () => {
it('keeps it out of provisioning and out of attachments', async () => {
/* These files are read inside primeResources, so the caller never sees them to
diff --git a/packages/api/src/agents/resources.ts b/packages/api/src/agents/resources.ts
index 0a5d6fadc4..513ab5e694 100644
--- a/packages/api/src/agents/resources.ts
+++ b/packages/api/src/agents/resources.ts
@@ -7,7 +7,7 @@ import {
FileSources,
getCodeEnvRefs,
canToolResourceConsume,
- hasToolResourceProvisioning,
+ hasToolResourceDestination,
} from 'librechat-data-provider';
import type {
AgentToolResources,
@@ -496,17 +496,8 @@ const computeProvisionState = async ({
return choice == null ? true : choice;
};
- /** A chooser upload may still be provisioned for the destination it was filed under,
- * which is how an inherited search file gets vectors in a duplicated agent's namespace.
- * The destinations it carries no reference for were declined, not deferred. */
- /** What the record itself shows about where it was sent. A message attachment is never
- * in the agent's resources, so a reference or an embedding is the only evidence that
- * the chooser picked that destination, and it is proof: nothing else creates one. */
- /* The predicate the turn's delivery decision reads too, so a file this queue has yet to
- * provision is never one the prompt withheld its text for. */
- const carriesEvidenceFor = (file: TFile, resourceType: EToolResources): boolean =>
- hasToolResourceProvisioning(file, resourceType);
-
+ /* Search intent survives vector cleanup without claiming the vectors still exist.
+ * A chooser's other destinations remain declined unless membership or a copy proves them. */
const allowsResource = (file: TFile, resourceType: EToolResources): boolean => {
if (!cameFromChooser(file)) {
return true;
@@ -514,7 +505,7 @@ const computeProvisionState = async ({
const isPersistedResource =
file.file_id != null &&
persistedResourceMembership?.get(resourceType)?.has(file.file_id) === true;
- return isPersistedResource || carriesEvidenceFor(file, resourceType);
+ return isPersistedResource || hasToolResourceDestination(file, resourceType);
};
const provisionable = attachments.filter(
(file) =>
diff --git a/packages/api/src/files/code/delete.spec.ts b/packages/api/src/files/code/delete.spec.ts
new file mode 100644
index 0000000000..2a3e4b34cf
--- /dev/null
+++ b/packages/api/src/files/code/delete.spec.ts
@@ -0,0 +1,141 @@
+import type { CodeEnvRef } from 'librechat-data-provider';
+import type { ServerRequest } from '~/types';
+import { createFileProvisionRecovery } from '../operations';
+import { createCodeFileDeleter } from './delete';
+
+const req = { user: { id: 'owner' } } as ServerRequest;
+const ref: CodeEnvRef = {
+ kind: 'user',
+ id: 'owner',
+ file_id: 'remote-copy',
+ storage_session_id: 'session',
+ provisionedAt: Date.now(),
+};
+
+function setup() {
+ const httpClient = jest.fn().mockResolvedValue({ status: 204 });
+ const clearFileCodeEnvRefs = jest.fn().mockResolvedValue(true);
+ const remove = createCodeFileDeleter({
+ httpClient,
+ clearFileCodeEnvRefs,
+ getAuthHeaders: jest.fn().mockResolvedValue({}),
+ getBaseUrl: jest.fn(() => 'https://code.example'),
+ routeKey: jest.fn(() => 'default'),
+ profileHeaders: jest.fn(() => ({})),
+ downloadQuery: jest.fn(() => '?kind=user&id=owner'),
+ logger: { warn: jest.fn() },
+ });
+ return { httpClient, clearFileCodeEnvRefs, remove };
+}
+
+it.each([204, 404])('reconciles a confirmed code deletion or absence (%s)', async (status) => {
+ const { httpClient, clearFileCodeEnvRefs, remove } = setup();
+ if (status === 404) httpClient.mockRejectedValue({ response: { status } });
+ const file = { file_id: 'stored-file', metadata: { codeEnvRefs: { default: ref } } };
+ await expect(remove(req, file)).resolves.toEqual({ deleted: true });
+ expect(clearFileCodeEnvRefs).toHaveBeenCalledWith({
+ file_id: 'stored-file',
+ userId: 'owner',
+ tenantId: undefined,
+ refs: [{ routeKey: 'default', ref }],
+ });
+ expect(file.metadata.codeEnvRefs.default).toBe(ref);
+});
+
+it('persists each confirmed route before a later route fails', async () => {
+ const { httpClient, clearFileCodeEnvRefs, remove } = setup();
+ httpClient
+ .mockResolvedValueOnce({ status: 204 })
+ .mockRejectedValueOnce(new Error('code offline'));
+ const other: CodeEnvRef = { ...ref, executionProfile: 'stateful', file_id: 'other-copy' };
+ await expect(
+ remove(req, {
+ file_id: 'stored-file',
+ metadata: { codeEnvRefs: { default: ref, stateful: other } },
+ }),
+ ).rejects.toThrow('code offline');
+ expect(clearFileCodeEnvRefs).toHaveBeenCalledTimes(1);
+ expect(clearFileCodeEnvRefs.mock.invocationCallOrder[0]).toBeLessThan(
+ httpClient.mock.invocationCallOrder[1],
+ );
+});
+
+it.each(['conflict', 'failure'])(
+ 'does not forgive code reconciliation %s as storage absence',
+ async (mode) => {
+ const { clearFileCodeEnvRefs, remove } = setup();
+ if (mode === 'conflict') clearFileCodeEnvRefs.mockResolvedValue(false);
+ else clearFileCodeEnvRefs.mockRejectedValue(new Error('resource not found'));
+ await expect(
+ remove(req, { file_id: 'stored-file', metadata: { codeEnvRef: ref } }),
+ ).rejects.toThrow('Code storage reconciliation failed');
+ },
+);
+
+it('does not confirm or invalidate an unmapped historical route', async () => {
+ const { httpClient, clearFileCodeEnvRefs, remove } = setup();
+ const route = 'stateful:0123456789abcdef0123456789abcdef';
+ await expect(
+ remove(req, {
+ file_id: 'stored-file',
+ metadata: {
+ codeEnvRefs: {
+ [route]: { ...ref, executionProfile: 'stateful', executionRouteKey: route },
+ },
+ },
+ }),
+ ).resolves.toBeUndefined();
+ expect(httpClient).not.toHaveBeenCalled();
+ expect(clearFileCodeEnvRefs).not.toHaveBeenCalled();
+});
+
+it('invalidates a confirmed code pointer even when a later vector deletion fails', async () => {
+ const { createDeleteFileWithSecondaryStorage } = await import('../storage');
+ const { FileSources } = await import('librechat-data-provider');
+ const { clearFileCodeEnvRefs, remove } = setup();
+ const retained = {
+ file_id: 'stored-file',
+ embedded: true,
+ metadata: { codeEnvRef: ref },
+ };
+ const primary = jest.fn();
+ const clearFileEmbeddings = jest.fn();
+ const deleteFile = createDeleteFileWithSecondaryStorage({
+ source: FileSources.local,
+ deleteFile: primary,
+ getDeleteFile: (source) =>
+ source === FileSources.execute_code
+ ? async () => remove(req, retained)
+ : async () => {
+ throw new Error('vector unavailable');
+ },
+ clearFileEmbeddings,
+ isFileStorageClean: jest.fn().mockResolvedValue(true),
+ fileStorage: {
+ recovery: createFileProvisionRecovery(),
+ acquireFileProvision: jest.fn().mockResolvedValue(true),
+ releaseFileProvision: jest.fn().mockResolvedValue(true),
+ commitFileProvision: jest.fn().mockResolvedValue(true),
+ acquireFileDeletion: jest.fn().mockResolvedValue({ resumed: false }),
+ releaseFileDeletion: jest.fn().mockResolvedValue(true),
+ },
+ logger: { warn: jest.fn() },
+ });
+ await expect(
+ deleteFile(
+ {
+ user: { id: 'owner' },
+ config: { fileConfig: { retainedFileCleanup: true } },
+ },
+ retained,
+ ),
+ ).rejects.toThrow('vector unavailable');
+ expect(clearFileCodeEnvRefs).toHaveBeenCalledWith(
+ expect.objectContaining({
+ file_id: 'stored-file',
+ refs: [{ routeKey: 'default', ref }],
+ }),
+ );
+ expect(primary).not.toHaveBeenCalled();
+ expect(clearFileEmbeddings).not.toHaveBeenCalled();
+});
diff --git a/packages/api/src/files/code/delete.ts b/packages/api/src/files/code/delete.ts
new file mode 100644
index 0000000000..d889855440
--- /dev/null
+++ b/packages/api/src/files/code/delete.ts
@@ -0,0 +1,131 @@
+import { EModelEndpoint, getCodeEnvRefs } from 'librechat-data-provider';
+import type { ClearFileCodeEnvRefsParams } from '@librechat/data-schemas';
+import type { CodeEnvReferenceSet } from 'librechat-data-provider';
+import type { AxiosRequestConfig } from 'axios';
+import type {
+ getCodeExecutionBaseUrl,
+ createCodeExecutionRouteKey,
+ codeExecutionHeaders,
+} from '~/agents/execution';
+import type { getCodeApiAuthHeaders } from '~/auth/codeapi';
+import type { buildCodeEnvDownloadQuery } from './identity';
+import type { ServerRequest } from '~/types';
+import { getSafeErrorMetadata } from '../../utils/errors';
+
+/** Deletes and reconciles each confirmed copy before another route can fail. */
+export function createCodeFileDeleter({
+ httpClient,
+ getAuthHeaders,
+ getBaseUrl,
+ routeKey,
+ profileHeaders,
+ downloadQuery,
+ clearFileCodeEnvRefs,
+ logger,
+ httpAgent,
+ httpsAgent,
+}: {
+ httpClient: (config: AxiosRequestConfig) => Promise<unknown>;
+ getAuthHeaders: typeof getCodeApiAuthHeaders;
+ getBaseUrl: typeof getCodeExecutionBaseUrl;
+ routeKey: typeof createCodeExecutionRouteKey;
+ profileHeaders: typeof codeExecutionHeaders;
+ downloadQuery: typeof buildCodeEnvDownloadQuery;
+ clearFileCodeEnvRefs: (data: ClearFileCodeEnvRefsParams) => Promise<boolean>;
+ logger: { warn: (message: string, metadata?: unknown) => void };
+ httpAgent?: AxiosRequestConfig['httpAgent'];
+ httpsAgent?: AxiosRequestConfig['httpsAgent'];
+}): (
+ req: ServerRequest,
+ file: {
+ file_id?: string;
+ user?: { toString(): string };
+ tenantId?: string;
+ metadata?: CodeEnvReferenceSet;
+ },
+) => Promise<{ deleted: true } | undefined> {
+ return async (req, file) => {
+ const refs = getCodeEnvRefs(file.metadata);
+ if (refs.length === 0) return undefined;
+ let skipped = false;
+ for (const [executionRouteKey, ref] of refs) {
+ const executionProfile = ref.executionProfile ?? 'default';
+ const environments =
+ req.config?.endpoints?.[EModelEndpoint.agents]?.statefulCodeSessions?.environments;
+ const environment = environments?.find(
+ (candidate) => routeKey(executionProfile, candidate) === executionRouteKey,
+ );
+ if (
+ executionProfile === 'stateful' &&
+ executionRouteKey !== executionProfile &&
+ !environment
+ ) {
+ skipped = true;
+ continue;
+ }
+ let baseUrl: string;
+ try {
+ baseUrl = getBaseUrl(executionProfile, environment);
+ } catch (error) {
+ if (
+ executionProfile !== 'stateful' ||
+ executionRouteKey !== executionProfile ||
+ environment
+ ) {
+ throw error;
+ }
+ skipped = true;
+ continue;
+ }
+ const query = downloadQuery({
+ kind: ref.kind,
+ id: ref.id,
+ ...(ref.kind === 'skill' ? { version: ref.version } : {}),
+ });
+ const bridgeWorkerId = environment?.workerId ?? environment?.pairing?.workerId;
+ const headers = await getAuthHeaders(req, bridgeWorkerId);
+ try {
+ await httpClient({
+ method: 'delete',
+ url: `${baseUrl}/files/${ref.storage_session_id}/${ref.file_id}${query}`,
+ headers: {
+ 'User-Agent': 'LibreChat/1.0',
+ ...headers,
+ ...profileHeaders({ executionProfile, bridgeWorkerId }),
+ },
+ httpAgent,
+ httpsAgent,
+ timeout: 15000,
+ });
+ } catch (error) {
+ const response = (error as { response?: { status?: number; data?: { error?: string } } })
+ ?.response;
+ if (response?.status !== 404) {
+ if (response?.status === 409 && response.data?.error === 'execution_profile_mismatch') {
+ skipped = true;
+ continue;
+ }
+ throw error;
+ }
+ logger.warn('Code storage copy already absent during delete', getSafeErrorMetadata(error));
+ }
+ if (file.file_id) {
+ try {
+ if (
+ !(await clearFileCodeEnvRefs({
+ file_id: file.file_id,
+ userId: file.user?.toString() ?? req.user?.id ?? '',
+ tenantId: file.tenantId,
+ refs: [{ routeKey: executionRouteKey, ref }],
+ }))
+ ) {
+ throw new Error('Code reference snapshot changed');
+ }
+ } catch (error) {
+ throw new Error('Code storage reconciliation failed', { cause: error });
+ }
+ }
+ }
+ return skipped ? undefined : { deleted: true };
+ };
+}
diff --git a/packages/api/src/files/code/index.ts b/packages/api/src/files/code/index.ts
index 31029d3315..f69ef2406b 100644
--- a/packages/api/src/files/code/index.ts
+++ b/packages/api/src/files/code/index.ts
@@ -1,6 +1,7 @@
export * from './classify';
export * from './artifactDelivery';
export * from './destinations';
+export * from './delete';
export * from './errors';
export * from './extract';
export * from './form';
diff --git a/packages/api/src/files/code/process.ts b/packages/api/src/files/code/process.ts
index 9aa4d1d970..e3709bb5c1 100644
--- a/packages/api/src/files/code/process.ts
+++ b/packages/api/src/files/code/process.ts
@@ -17,6 +17,7 @@ import type {
CodeOutputDownloadFallback,
ProcessPublishedCodeOutputInput,
} from './publication';
+import type { FileProvisionStorage, FileStorageUploadObserver } from '../operations';
import type { sanitizeArtifactPath, flattenArtifactPath } from '~/utils/files';
import type { classifyCodeArtifact, CodeArtifactCategory } from './classify';
import type { hasOfficeHtmlPath, getExtractedTextFormat } from './extract';
@@ -24,6 +25,7 @@ import type { RetentionExpiry } from '~/files/retention';
import type { extractCodeArtifactText } from './extract';
import type { logAxiosError } from '~/utils/axios';
import type { ServerRequest } from '~/types';
+import { withFileProvisioning, FileProvisionPersistenceError } from '../operations';
import { createCodeOutputPersistence } from './publication';
export type CodeOutputProcessorInput = Omit<
@@ -57,6 +59,7 @@ type DownloadFallbackInput = Pick<
> & { expiresAt: number };
export interface CodeOutputProcessorDeps {
+ fileStorage: FileProvisionStorage;
getCodeOutputFileSettings: (req: ServerRequest) => {
endpointFileConfig: ReturnType<typeof getEndpointFileConfig>;
fileSizeLimit: number;
@@ -140,7 +143,7 @@ export async function processCodeOutput(
getStrategyFunctions,
determineFileType,
claimCodeFile,
- commitCodeFile: commitOutput,
+ fileStorage,
finalizePreview,
hasOfficeHtmlPath,
sanitizeArtifactPath,
@@ -299,282 +302,297 @@ export async function processCodeOutput(
);
}
- /**
- * The data-schema method checks background dispatch ownership atomically with
- * the write. A superseded commit misses and its attachment is skipped.
- * Bytes already uploaded to a reused storage key retain the existing race;
- * a per-file lock would be needed to cover that storage write as well.
- * Foreground outputs retain their unconditional commit behavior.
- */
+ /** Retain the completed output for token-qualified persistence after storage settles. */
+ let pendingOutput: CodeOutputStoredFile | undefined;
async function commitCodeFile(fileData: CodeOutputStoredFile): Promise<boolean> {
- const committed = await commitOutput(
- fileData,
- freshClaimAfter == null ? undefined : sourceDispatchedAt,
- );
- if (!committed) {
+ pendingOutput = structuredClone(fileData);
+ return true;
+ }
+
+ const writeOutput = async (
+ onUploadStart: FileStorageUploadObserver,
+ ): Promise<CodeOutputResult | null> => {
+ /**
+ * Preserve the original `messageId` on update. Each `processCodeOutput`
+ * call would otherwise overwrite it with the current run's run id, which
+ * decouples the file from the assistant message that originally created
+ * it. `getCodeGeneratedFiles` filters by `messageId IN <thread>`, so a
+ * stale id (e.g. from a later regeneration / failed re-read attempt)
+ * silently excludes the file from priming on subsequent turns.
+ */
+ const persistedMessageId = isUpdate ? (claimed.messageId ?? messageId) : messageId;
+ /* A generated-output write replaces the file's bytes, so pointers to
+ * earlier content in another profile must not survive as reusable refs. */
+ const codeEnvReferenceSet = mergeCodeEnvRef(undefined, codeEnvRef);
+ const codeEnvMetadata = {
+ ...claimed.metadata,
+ ...codeEnvReferenceSet,
+ sourceDispatchedAt,
+ };
+
+ if (isImage) {
+ const usage = isUpdate ? (claimed.usage ?? 0) + 1 : 1;
+ onUploadStart();
+ const _file = await convertImage(req, buffer, 'high', `${file_id}${fileExt}`);
+ const filepath = usage > 1 ? `${_file.filepath}?v=${Date.now()}` : _file.filepath;
+ const storageMetadata = getStorageMetadata({
+ filepath: _file.filepath,
+ source: appConfig.fileStrategy,
+ storageKey: _file.storageKey,
+ storageRegion: _file.storageRegion,
+ });
+ outputPersistence.trackStored({
+ ..._file,
+ ...storageMetadata,
+ file_id,
+ filepath: _file.filepath,
+ filename: safeName,
+ type: `image/${appConfig.imageOutputType}`,
+ user: req.user.id,
+ tenantId: req.user.tenantId,
+ conversationId,
+ source: appConfig.fileStrategy,
+ });
+ const file: CodeOutputStoredFile & Pick<CodeEnvRef, 'executionProfile'> = {
+ ..._file,
+ filepath,
+ ...storageMetadata,
+ file_id,
+ messageId: persistedMessageId,
+ usage,
+ filename: safeName,
+ conversationId,
+ executionProfile,
+ user: req.user.id,
+ tenantId: req.user.tenantId,
+ type: `image/${appConfig.imageOutputType}`,
+ createdAt: isUpdate ? claimed.createdAt : formattedDate,
+ updatedAt: formattedDate,
+ source: appConfig.fileStrategy,
+ context: outputPersistence.context,
+ metadata: codeEnvMetadata,
+ ...(await retentionExpiryPromise),
+ };
+ if (!(await outputPersistence.commit(file))) {
+ return null;
+ }
+ return { file: Object.assign(file, { messageId, toolCallId, agentId }) };
+ }
+
+ const { saveBuffer } = getStrategyFunctions(appConfig.fileStrategy);
+ if (!saveBuffer) {
logger.warn(
- `[processCodeOutput] Skipping stale background output "${safeName}" (${file_id}): a newer run owns this filename`,
+ `[processCodeOutput] saveBuffer not available for strategy ${appConfig.fileStrategy}, falling back to download URL`,
);
+ return {
+ file: createDownloadFallback({
+ id,
+ name,
+ agentId,
+ messageId,
+ toolCallId,
+ session_id,
+ conversationId,
+ executionProfile,
+ executionRouteKey,
+ expiresAt: currentDate.getTime() + 86400000,
+ }),
+ };
}
- return committed;
- }
- /**
- * Preserve the original `messageId` on update. Each `processCodeOutput`
- * call would otherwise overwrite it with the current run's run id, which
- * decouples the file from the assistant message that originally created
- * it. `getCodeGeneratedFiles` filters by `messageId IN <thread>`, so a
- * stale id (e.g. from a later regeneration / failed re-read attempt)
- * silently excludes the file from priming on subsequent turns.
- */
- const persistedMessageId = isUpdate ? (claimed.messageId ?? messageId) : messageId;
- /* A generated-output write replaces the file's bytes, so pointers to
- * earlier content in another profile must not survive as reusable refs. */
- const codeEnvReferenceSet = mergeCodeEnvRef(undefined, codeEnvRef);
- const codeEnvMetadata = {
- ...claimed.metadata,
- ...codeEnvReferenceSet,
- sourceDispatchedAt,
- };
+ const detectedType = await determineFileType(buffer, true);
+ const mimeType = detectedType?.mime || inferMimeType(name, '') || 'application/octet-stream';
+
+ /** Check MIME type support - for code-generated files, we're lenient but log unsupported types */
+ const isSupportedMimeType = fileConfig.checkType(
+ mimeType,
+ endpointFileConfig.supportedMimeTypes,
+ );
+ if (!isSupportedMimeType) {
+ logger.warn(
+ `[processCodeOutput] File "${name}" has unsupported MIME type "${mimeType}", proceeding with storage but may not be usable as tool resource`,
+ );
+ }
- if (isImage) {
- const usage = isUpdate ? (claimed.usage ?? 0) + 1 : 1;
- const _file = await convertImage(req, buffer, 'high', `${file_id}${fileExt}`);
- const filepath = usage > 1 ? `${_file.filepath}?v=${Date.now()}` : _file.filepath;
+ /* Compose the storage key here, after `file_id` is known, so the
+ * `flattenArtifactPath` cap budget can be calculated against the
+ * actual prefix length. The full key has to fit in one filesystem
+ * path component (NAME_MAX = 255 on most filesystems); without this
+ * cap, deeply-nested artifact paths whose individual segments were
+ * within bounds can still produce a flat form that overflows once
+ * `${file_id}__` is prepended, causing `ENAMETOOLONG` inside
+ * saveBuffer and falling back to a download URL. The 255 figure is
+ * the conservative cross-platform NAME_MAX (Linux ext4, NTFS, APFS).
+ */
+ const NAME_MAX = 255;
+ const flatName = flattenArtifactPath(safeName, NAME_MAX - file_id.length - 2);
+ const fileName = `${file_id}__${flatName}`;
+ onUploadStart();
+ const filepath = await saveBuffer({
+ userId: req.user.id,
+ buffer,
+ fileName,
+ basePath: 'uploads',
+ tenantId: req.user.tenantId,
+ });
const storageMetadata = getStorageMetadata({
- filepath: _file.filepath,
+ filepath,
source: appConfig.fileStrategy,
- storageKey: _file.storageKey,
- storageRegion: _file.storageRegion,
});
outputPersistence.trackStored({
- ..._file,
- ...storageMetadata,
file_id,
- filepath: _file.filepath,
+ filepath,
+ ...storageMetadata,
filename: safeName,
- type: `image/${appConfig.imageOutputType}`,
+ type: mimeType,
+ bytes: buffer.length,
user: req.user.id,
tenantId: req.user.tenantId,
conversationId,
source: appConfig.fileStrategy,
});
- const file: CodeOutputStoredFile & Pick<CodeEnvRef, 'executionProfile'> = {
- ..._file,
+
+ /* `classifyCodeArtifact` and `extractCodeArtifactText` make
+ * extension/bare-name decisions on the input string. With the
+ * path-preserving sanitizer they can now receive a nested path like
+ * `reports.v1/Makefile`, which the classifier's `extensionOf` reads
+ * as `v1/Makefile` (the slice after the dot in the directory name)
+ * and the bare-name branch rejects because it sees a `.` anywhere in
+ * the string. Result: extensionless artifacts under dotted folders
+ * (Makefile, Dockerfile, etc.) get misclassified as `other` and
+ * skip text extraction. Pass the basename so classification matches
+ * what it would have gotten with the old flat-name flow. */
+ const leafName = path.basename(safeName);
+ const category = classifyCodeArtifact(leafName, mimeType);
+
+ /* Office-bucket files (DOCX/XLSX/XLS/ODS/CSV/PPTX) route through
+ * `bufferToOfficeHtml` which is CPU-heavy. Persist the record now
+ * with `status: 'pending'` and `text: null` so the agent's response
+ * isn't blocked, then return a `finalize` thunk the caller can run
+ * in the background. Non-office files have cheap or no extraction
+ * — run it inline so the caller gets a fully-resolved record
+ * without juggling a finalize step. */
+ const expectsPreview = hasOfficeHtmlPath(leafName, mimeType);
+
+ const baseFile: CodeOutputStoredFile = {
+ file_id,
filepath,
...storageMetadata,
- file_id,
messageId: persistedMessageId,
- usage,
+ object: 'file',
filename: safeName,
+ type: mimeType,
conversationId,
- executionProfile,
user: req.user.id,
tenantId: req.user.tenantId,
- type: `image/${appConfig.imageOutputType}`,
- createdAt: isUpdate ? claimed.createdAt : formattedDate,
+ bytes: buffer.length,
updatedAt: formattedDate,
+ metadata: codeEnvMetadata,
source: appConfig.fileStrategy,
context: outputPersistence.context,
- metadata: codeEnvMetadata,
+ usage: isUpdate ? (claimed.usage ?? 0) + 1 : 1,
+ createdAt: isUpdate ? claimed.createdAt : formattedDate,
...(await retentionExpiryPromise),
};
- if (!(await outputPersistence.commit(file))) {
- return null;
- }
- return { file: Object.assign(file, { messageId, toolCallId, agentId }) };
- }
-
- const { saveBuffer } = getStrategyFunctions(appConfig.fileStrategy);
- if (!saveBuffer) {
- logger.warn(
- `[processCodeOutput] saveBuffer not available for strategy ${appConfig.fileStrategy}, falling back to download URL`,
- );
- return {
- file: createDownloadFallback({
- id,
- name,
- agentId,
- messageId,
- toolCallId,
- session_id,
- conversationId,
- executionProfile,
- executionRouteKey,
- expiresAt: currentDate.getTime() + 86400000,
- }),
- };
- }
-
- const detectedType = await determineFileType(buffer, true);
- const mimeType = detectedType?.mime || inferMimeType(name, '') || 'application/octet-stream';
-
- /** Check MIME type support - for code-generated files, we're lenient but log unsupported types */
- const isSupportedMimeType = fileConfig.checkType(
- mimeType,
- endpointFileConfig.supportedMimeTypes,
- );
- if (!isSupportedMimeType) {
- logger.warn(
- `[processCodeOutput] File "${name}" has unsupported MIME type "${mimeType}", proceeding with storage but may not be usable as tool resource`,
- );
- }
-
- /* Compose the storage key here, after `file_id` is known, so the
- * `flattenArtifactPath` cap budget can be calculated against the
- * actual prefix length. The full key has to fit in one filesystem
- * path component (NAME_MAX = 255 on most filesystems); without this
- * cap, deeply-nested artifact paths whose individual segments were
- * within bounds can still produce a flat form that overflows once
- * `${file_id}__` is prepended, causing `ENAMETOOLONG` inside
- * saveBuffer and falling back to a download URL. The 255 figure is
- * the conservative cross-platform NAME_MAX (Linux ext4, NTFS, APFS).
- */
- const NAME_MAX = 255;
- const flatName = flattenArtifactPath(safeName, NAME_MAX - file_id.length - 2);
- const fileName = `${file_id}__${flatName}`;
- const filepath = await saveBuffer({
- userId: req.user.id,
- buffer,
- fileName,
- basePath: 'uploads',
- tenantId: req.user.tenantId,
- });
- const storageMetadata = getStorageMetadata({
- filepath,
- source: appConfig.fileStrategy,
- });
- outputPersistence.trackStored({
- file_id,
- filepath,
- ...storageMetadata,
- filename: safeName,
- type: mimeType,
- bytes: buffer.length,
- user: req.user.id,
- tenantId: req.user.tenantId,
- conversationId,
- source: appConfig.fileStrategy,
- });
- /* `classifyCodeArtifact` and `extractCodeArtifactText` make
- * extension/bare-name decisions on the input string. With the
- * path-preserving sanitizer they can now receive a nested path like
- * `reports.v1/Makefile`, which the classifier's `extensionOf` reads
- * as `v1/Makefile` (the slice after the dot in the directory name)
- * and the bare-name branch rejects because it sees a `.` anywhere in
- * the string. Result: extensionless artifacts under dotted folders
- * (Makefile, Dockerfile, etc.) get misclassified as `other` and
- * skip text extraction. Pass the basename so classification matches
- * what it would have gotten with the old flat-name flow. */
- const leafName = path.basename(safeName);
- const category = classifyCodeArtifact(leafName, mimeType);
-
- /* Office-bucket files (DOCX/XLSX/XLS/ODS/CSV/PPTX) route through
- * `bufferToOfficeHtml` which is CPU-heavy. Persist the record now
- * with `status: 'pending'` and `text: null` so the agent's response
- * isn't blocked, then return a `finalize` thunk the caller can run
- * in the background. Non-office files have cheap or no extraction
- * — run it inline so the caller gets a fully-resolved record
- * without juggling a finalize step. */
- const expectsPreview = hasOfficeHtmlPath(leafName, mimeType);
-
- const baseFile: CodeOutputStoredFile = {
- file_id,
- filepath,
- ...storageMetadata,
- messageId: persistedMessageId,
- object: 'file',
- filename: safeName,
- type: mimeType,
- conversationId,
- user: req.user.id,
- tenantId: req.user.tenantId,
- bytes: buffer.length,
- updatedAt: formattedDate,
- metadata: codeEnvMetadata,
- source: appConfig.fileStrategy,
- context: outputPersistence.context,
- usage: isUpdate ? (claimed.usage ?? 0) + 1 : 1,
- createdAt: isUpdate ? claimed.createdAt : formattedDate,
- ...(await retentionExpiryPromise),
- };
+ if (expectsPreview) {
+ /* Persist with `status: 'pending'` and explicit
+ * `text: null` / `textFormat: null` so an update that previously
+ * had cached text gets cleared. The deferred finalize transitions
+ * to 'ready' (with text/textFormat) or 'failed' (with
+ * previewError).
+ *
+ * `previewRevision` is a fresh UUID stamped on every emit. The
+ * deferred finalize's `updateFile` is conditional on this — if
+ * a newer turn (cross-turn filename reuse) has rotated the
+ * revision before this render finishes, the stale render is
+ * silently discarded rather than overwriting the newer record.
+ * (Codex P1 review on PR #12957.) */
+ const previewRevision = v4();
+ const file = {
+ ...baseFile,
+ text: null,
+ textFormat: null,
+ status: 'pending',
+ previewError: null,
+ previewRevision,
+ } satisfies CodeOutputStoredFile;
+ if (!(await outputPersistence.commit(file))) {
+ return null;
+ }
+ return {
+ file: Object.assign(file, { messageId, toolCallId, agentId }),
+ finalize: outputPersistence.finalize(file, () =>
+ finalizePreview({
+ buffer,
+ leafName,
+ mimeType,
+ category,
+ file_id: file.file_id,
+ previewRevision: file.previewRevision,
+ }),
+ ),
+ previewRevision: file.previewRevision,
+ };
+ }
- if (expectsPreview) {
- /* Persist with `status: 'pending'` and explicit
- * `text: null` / `textFormat: null` so an update that previously
- * had cached text gets cleared. The deferred finalize transitions
- * to 'ready' (with text/textFormat) or 'failed' (with
- * previewError).
- *
- * `previewRevision` is a fresh UUID stamped on every emit. The
- * deferred finalize's `updateFile` is conditional on this — if
- * a newer turn (cross-turn filename reuse) has rotated the
- * revision before this render finishes, the stale render is
- * silently discarded rather than overwriting the newer record.
- * (Codex P1 review on PR #12957.) */
- const previewRevision = v4();
- const file = {
+ /* Non-office path: extraction is cheap (utf8 decode, parseDocument
+ * for PDF/ODT, or null for binaries). Run inline and return a
+ * fully-resolved record — no `finalize` needed. */
+ const text = await extractCodeArtifactText(buffer, leafName, mimeType, category);
+ /* `textFormat` accompanies `text` so the client can gate
+ * office-HTML-bucket routing on a trusted signal — clients MUST
+ * NOT inject `text` into the iframe as HTML unless `textFormat ===
+ * 'html'`. RAG-uploaded `.docx` etc. arrive with plain text from
+ * mammoth.extractRawText and would otherwise be hijacked by the
+ * extension-based office routing into the HTML-injection path
+ * (Codex P1 review on PR #12934). null on extract failure — the
+ * client treats absence as 'text' for safety. */
+ const textFormat = getExtractedTextFormat(leafName, mimeType, text);
+ const file: CodeOutputStoredFile = {
...baseFile,
- text: null,
- textFormat: null,
- status: 'pending',
+ // Always set explicitly so an update which produces a binary or
+ // oversized artifact clears any previously cached text — createFile
+ // uses findOneAndUpdate with $set semantics.
+ text: text ?? null,
+ textFormat: textFormat ?? null,
+ // Clear deferred-preview lifecycle fields in case the prior emit
+ // at this (filename, conversationId) was an office file —
+ // otherwise stale `pending`/`failed` would persist and the client
+ // would render the wrong state for the now non-office artifact.
+ status: null,
previewError: null,
- previewRevision,
- } satisfies CodeOutputStoredFile;
+ previewRevision: null,
+ };
+
if (!(await outputPersistence.commit(file))) {
return null;
}
- return {
- file: Object.assign(file, { messageId, toolCallId, agentId }),
- finalize: outputPersistence.finalize(file, () =>
- finalizePreview({
- buffer,
- leafName,
- mimeType,
- category,
- file_id: file.file_id,
- previewRevision: file.previewRevision,
- }),
- ),
- previewRevision: file.previewRevision,
- };
- }
-
- /* Non-office path: extraction is cheap (utf8 decode, parseDocument
- * for PDF/ODT, or null for binaries). Run inline and return a
- * fully-resolved record — no `finalize` needed. */
- const text = await extractCodeArtifactText(buffer, leafName, mimeType, category);
- /* `textFormat` accompanies `text` so the client can gate
- * office-HTML-bucket routing on a trusted signal — clients MUST
- * NOT inject `text` into the iframe as HTML unless `textFormat ===
- * 'html'`. RAG-uploaded `.docx` etc. arrive with plain text from
- * mammoth.extractRawText and would otherwise be hijacked by the
- * extension-based office routing into the HTML-injection path
- * (Codex P1 review on PR #12934). null on extract failure — the
- * client treats absence as 'text' for safety. */
- const textFormat = getExtractedTextFormat(leafName, mimeType, text);
- const file: CodeOutputStoredFile = {
- ...baseFile,
- // Always set explicitly so an update which produces a binary or
- // oversized artifact clears any previously cached text — createFile
- // uses findOneAndUpdate with $set semantics.
- text: text ?? null,
- textFormat: textFormat ?? null,
- // Clear deferred-preview lifecycle fields in case the prior emit
- // at this (filename, conversationId) was an office file —
- // otherwise stale `pending`/`failed` would persist and the client
- // would render the wrong state for the now non-office artifact.
- status: null,
- previewError: null,
- previewRevision: null,
+ return { file: Object.assign(file, { messageId, toolCallId, agentId }) };
};
-
- if (!(await outputPersistence.commit(file))) {
- return null;
- }
- return { file: Object.assign(file, { messageId, toolCallId, agentId }) };
+ if (publication) return await writeOutput(() => undefined);
+ return await withFileProvisioning({
+ file: { file_id, user: req.user.id, tenantId: req.user.tenantId },
+ userId: req.user.id,
+ storage: fileStorage,
+ destination: JSON.stringify(['generated-output', sourceDispatchedAt, id]),
+ operation: async (_claim, onUploadStart) => {
+ const value = await writeOutput(onUploadStart);
+ return {
+ value,
+ commit: pendingOutput
+ ? {
+ output: {
+ file: pendingOutput,
+ sourceDispatchedAt: freshClaimAfter == null ? undefined : sourceDispatchedAt,
+ },
+ }
+ : { releaseOnly: true },
+ };
+ },
+ });
} catch (error) {
+ if (freshClaimAfter != null && error instanceof FileProvisionPersistenceError) return null;
if (
error != null &&
typeof error === 'object' &&
diff --git a/packages/api/src/files/dispatch.spec.ts b/packages/api/src/files/dispatch.spec.ts
new file mode 100644
index 0000000000..648d66f659
--- /dev/null
+++ b/packages/api/src/files/dispatch.spec.ts
@@ -0,0 +1,142 @@
+import type { FileStorageClaim } from '@librechat/data-schemas';
+import type { FileStorageUploadObserver } from './operations';
+import {
+ createCodeApiRateLimitBudget,
+ createCodeApiUploadRegistry,
+ withCodeApiUploadRecovery,
+} from '../utils/code';
+import { createFileProvisionRecovery, withFileDeletion, withFileProvisioning } from './operations';
+import { dispatchFileDeletion, dispatchFileStorageOperation } from './dispatch';
+
+function fixture() {
+ const claims = new Set<string>();
+ const storage = {
+ recovery: createFileProvisionRecovery(),
+ acquireFileProvision: jest.fn(async ({ token }: { token: string }) => {
+ claims.add(token);
+ return true;
+ }),
+ releaseFileProvision: jest.fn(async ({ token }: { token: string }) => {
+ claims.delete(token);
+ return true;
+ }),
+ commitFileProvision: jest.fn(async () => true),
+ acquireFileDeletion: jest.fn(async () => (claims.size ? null : { resumed: false })),
+ releaseFileDeletion: jest.fn(async () => true),
+ };
+ return { storage, file: { file_id: 'file', user: 'owner' }, claims };
+}
+
+it('marks each definitive rejection without treating a later canceled wait as an upload', async () => {
+ const { storage, file, claims } = fixture();
+ const controller = new AbortController();
+ const rejected = Object.assign(new Error('throttled'), {
+ isAxiosError: true,
+ response: { status: 429, headers: { 'retry-after': '1' } },
+ });
+ const remote = jest.fn(async () => {
+ throw rejected;
+ });
+ const operation = jest.fn(
+ async (_claim: FileStorageClaim, onUploadStart: FileStorageUploadObserver) => {
+ await withCodeApiUploadRecovery({
+ registry: createCodeApiUploadRegistry(),
+ scope: 'route:user',
+ label: 'upload',
+ budget: createCodeApiRateLimitBudget(10000),
+ signal: controller.signal,
+ onWait: () => controller.abort(),
+ openSource: async () => 'source',
+ upload: () => dispatchFileStorageOperation(onUploadStart, remote),
+ });
+ return { value: 'uploaded', commit: { embedded: true } };
+ },
+ );
+ await expect(
+ withFileProvisioning({ file, userId: 'owner', storage, destination: 'code', operation }),
+ ).rejects.toMatchObject({ name: 'AbortError' });
+ expect(remote).toHaveBeenCalledTimes(1);
+ expect(claims.size).toBe(0);
+ await expect(
+ withFileDeletion({ file, userId: 'owner', storage, operation: async () => undefined }),
+ ).resolves.toBeUndefined();
+});
+
+it('retains a later uncertain request even after an earlier rejected attempt', async () => {
+ const { storage, file, claims } = fixture();
+ const denied = Object.assign(new Error('throttled'), { response: { status: 429 } });
+ await expect(
+ withFileProvisioning({
+ file,
+ userId: 'owner',
+ storage,
+ destination: 'code',
+ operation: async (_claim, onStart) => {
+ await dispatchFileStorageOperation(onStart, async () => {
+ throw denied;
+ }).catch(() => undefined);
+ await dispatchFileStorageOperation(onStart, async () => {
+ throw new Error('response lost');
+ });
+ return { value: 'unused', commit: { embedded: true } };
+ },
+ }),
+ ).rejects.toMatchObject({ name: 'FileProvisionOutcomeUnknownError' });
+ expect(claims.size).toBe(1);
+ expect(storage.releaseFileProvision).not.toHaveBeenCalled();
+});
+
+it('does not clear uncertainty after a successful transport response and failed result validation', async () => {
+ const { storage, file, claims } = fixture();
+ await expect(
+ withFileProvisioning({
+ file,
+ userId: 'owner',
+ storage,
+ destination: 'code',
+ operation: async (_claim, onStart) => {
+ await dispatchFileStorageOperation(onStart, async () => ({ status: 200 }));
+ throw new Error('malformed upload result');
+ },
+ }),
+ ).rejects.toMatchObject({ name: 'FileProvisionOutcomeUnknownError' });
+ expect(claims.size).toBe(1);
+});
+
+describe('dispatchFileDeletion', () => {
+ it('marks the mutation before dispatch and confirms its result', async () => {
+ const onDelete = jest.fn();
+ const operation = jest.fn(async () => {
+ expect(onDelete.mock.calls).toEqual([[]]);
+ return 'deleted';
+ });
+ await expect(dispatchFileDeletion(onDelete, operation)).resolves.toBe('deleted');
+ expect(onDelete.mock.calls).toEqual([[], ['confirmed']]);
+ });
+
+ it('reports a definitive mutation rejection', async () => {
+ const onDelete = jest.fn();
+ const error = Object.assign(new Error('denied'), { $metadata: { httpStatusCode: 403 } });
+ await expect(
+ dispatchFileDeletion(onDelete, async () => {
+ throw error;
+ }),
+ ).rejects.toBe(error);
+ expect(onDelete.mock.calls).toEqual([[], ['rejected']]);
+ });
+
+ it('leaves a dispatched mutation uncertain on a lost response', async () => {
+ const onDelete = jest.fn();
+ const error = Object.assign(new Error('response lost'), { code: 'ETIMEDOUT' });
+ await expect(
+ dispatchFileDeletion(onDelete, async () => {
+ throw error;
+ }),
+ ).rejects.toBe(error);
+ expect(onDelete.mock.calls).toEqual([[]]);
+ });
+
+ it('supports callers without an observer', async () => {
+ await expect(dispatchFileDeletion(undefined, async () => true)).resolves.toBe(true);
+ });
+});
diff --git a/packages/api/src/files/dispatch.ts b/packages/api/src/files/dispatch.ts
new file mode 100644
index 0000000000..58230f71e7
--- /dev/null
+++ b/packages/api/src/files/dispatch.ts
@@ -0,0 +1,34 @@
+import type { FileStorageUploadObserver } from './operations';
+import { isDefinitiveFileStorageFailure } from './operations';
+
+/** Definitive rejection closes this attempt before an outer retry can wait or cancel. */
+export async function dispatchFileStorageOperation<T>(
+ onStart: FileStorageUploadObserver | undefined,
+ operation: () => Promise<T>,
+): Promise<T> {
+ onStart?.();
+ try {
+ return await operation();
+ } catch (error) {
+ if (isDefinitiveFileStorageFailure(error)) onStart?.('rejected');
+ throw error;
+ }
+}
+
+export type FileStorageDeletionObserver = (outcome?: 'confirmed' | 'rejected') => void;
+
+/** Mark only the mutation, after provider preflight has completed. */
+export async function dispatchFileDeletion<T>(
+ onDelete: FileStorageDeletionObserver | undefined,
+ operation: () => Promise<T>,
+): Promise<T> {
+ onDelete?.();
+ try {
+ const result = await operation();
+ onDelete?.('confirmed');
+ return result;
+ } catch (error) {
+ if (isDefinitiveFileStorageFailure(error)) onDelete?.('rejected');
+ throw error;
+ }
+}
diff --git a/packages/api/src/files/finalize.spec.ts b/packages/api/src/files/finalize.spec.ts
new file mode 100644
index 0000000000..35d5fc63bb
--- /dev/null
+++ b/packages/api/src/files/finalize.spec.ts
@@ -0,0 +1,105 @@
+import { finalizeFileDeletion } from './finalize';
+
+const files = [{ file_id: 'gone' }, { file_id: 'kept' }];
+const params = { deletedFileIds: ['gone', 'kept'], failedFileIds: [], files, userId: 'owner' };
+function setup() {
+ const deleteFiles = jest.fn().mockResolvedValue({
+ deletedCount: 1,
+ deletedFileIds: ['gone'],
+ failedFileIds: ['kept'],
+ });
+ const removeAgentResourceFilesFromAllAgents = jest.fn().mockResolvedValue(undefined);
+ const logger = { error: jest.fn() };
+ const deps = { deleteFiles, removeAgentResourceFilesFromAllAgents, logger };
+ return deps;
+}
+
+it('cleans only successful file references after a mixed-snapshot metadata batch', async () => {
+ const deps = setup();
+ expect(await finalizeFileDeletion(params, deps)).toEqual({
+ deletedFileIds: ['gone'],
+ failedFileIds: ['kept'],
+ });
+ expect(deps.deleteFiles).toHaveBeenCalledWith(['gone', 'kept'], undefined, {
+ files,
+ userId: 'owner',
+ });
+ expect(deps.removeAgentResourceFilesFromAllAgents).toHaveBeenCalledWith({ file_ids: ['gone'] });
+});
+
+it('does not finalize any file whose storage cleanup failed', async () => {
+ const deps = setup();
+ deps.deleteFiles.mockResolvedValue({ deletedFileIds: ['gone'], failedFileIds: [] });
+ expect(await finalizeFileDeletion({ ...params, failedFileIds: ['kept'] }, deps)).toEqual({
+ deletedFileIds: ['gone'],
+ failedFileIds: ['kept'],
+ });
+ expect(deps.deleteFiles).toHaveBeenCalledWith(['gone'], undefined, expect.anything());
+});
+
+it('returns unresolved metadata ids as failures instead of silently claiming success', async () => {
+ const deps = setup();
+ deps.deleteFiles.mockResolvedValue({ deletedFileIds: ['gone'], failedFileIds: [] });
+ expect(await finalizeFileDeletion(params, deps)).toEqual({
+ deletedFileIds: ['gone'],
+ failedFileIds: ['kept'],
+ });
+});
+
+it('finalizes references for idempotently confirmed absent metadata', async () => {
+ const deps = setup();
+ deps.deleteFiles.mockResolvedValue({
+ deletedCount: 0,
+ deletedFileIds: ['gone', 'kept'],
+ failedFileIds: [],
+ });
+ expect(await finalizeFileDeletion(params, deps)).toEqual({
+ deletedFileIds: ['gone', 'kept'],
+ failedFileIds: [],
+ });
+ expect(deps.removeAgentResourceFilesFromAllAgents).toHaveBeenCalledWith({
+ file_ids: ['gone', 'kept'],
+ });
+});
+
+it('does not read or finalize metadata when no storage deletion succeeded', async () => {
+ const deps = setup();
+ expect(
+ await finalizeFileDeletion({ ...params, deletedFileIds: [], failedFileIds: ['kept'] }, deps),
+ ).toEqual({ deletedFileIds: [], failedFileIds: ['kept'] });
+ expect(deps.deleteFiles).not.toHaveBeenCalled();
+ expect(deps.removeAgentResourceFilesFromAllAgents).not.toHaveBeenCalled();
+});
+
+it('propagates metadata outages without removing agent references', async () => {
+ const deps = setup();
+ const failure = new Error('private database detail');
+ deps.deleteFiles.mockRejectedValue(failure);
+ await expect(finalizeFileDeletion(params, deps)).rejects.toBe(failure);
+ expect(deps.removeAgentResourceFilesFromAllAgents).not.toHaveBeenCalled();
+ expect(JSON.stringify(deps.logger.error.mock.calls)).not.toContain('private database detail');
+});
+
+it.each([
+ { deletedCount: 1 },
+ { deletedFileIds: ['unknown'], failedFileIds: [] },
+ { deletedFileIds: ['gone'], failedFileIds: ['gone'] },
+ { deletedFileIds: [], failedFileIds: ['unknown'] },
+])('rejects an unconfirmed or contradictory metadata receipt: %p', async (receipt) => {
+ const deps = setup();
+ deps.deleteFiles.mockResolvedValue(receipt);
+ await expect(finalizeFileDeletion(params, deps)).rejects.toThrow();
+ expect(deps.removeAgentResourceFilesFromAllAgents).not.toHaveBeenCalled();
+});
+
+it('keeps best-effort agent cleanup failure distinct from confirmed metadata deletion', async () => {
+ const deps = setup();
+ deps.removeAgentResourceFilesFromAllAgents.mockRejectedValue(
+ new Error('private database detail'),
+ );
+ expect(await finalizeFileDeletion(params, deps)).toEqual({
+ deletedFileIds: ['gone'],
+ failedFileIds: ['kept'],
+ });
+ expect(JSON.stringify(deps.logger.error.mock.calls)).not.toContain('private database detail');
+});
diff --git a/packages/api/src/files/finalize.ts b/packages/api/src/files/finalize.ts
new file mode 100644
index 0000000000..a345fbf24f
--- /dev/null
+++ b/packages/api/src/files/finalize.ts
@@ -0,0 +1,75 @@
+import type {
+ DeleteFilesOptions,
+ DeleteFilesResult,
+ FileDeletionSnapshot,
+} from '@librechat/data-schemas';
+import type { FileDeletionOutcome } from './deletion';
+import { getSafeErrorMetadata } from '../utils/errors';
+
+/** Finalizes only confirmed metadata deletions, preserving failed snapshots for retry. */
+export async function finalizeFileDeletion(
+ {
+ deletedFileIds,
+ failedFileIds,
+ files,
+ userId,
+ }: {
+ deletedFileIds: string[];
+ failedFileIds: string[];
+ files: FileDeletionSnapshot[];
+ userId: string;
+ },
+ {
+ deleteFiles,
+ removeAgentResourceFilesFromAllAgents,
+ logger,
+ }: {
+ deleteFiles: (
+ ids: string[],
+ user?: string,
+ options?: DeleteFilesOptions,
+ ) => Promise<DeleteFilesResult>;
+ removeAgentResourceFilesFromAllAgents: (params: { file_ids: string[] }) => Promise<unknown>;
+ logger: { error: (message: string, metadata: unknown) => void };
+ },
+): Promise<Required<FileDeletionOutcome>> {
+ const failed = new Set(failedFileIds);
+ const pending = [...new Set(deletedFileIds)].filter((id) => !failed.has(id));
+ if (pending.length === 0) return { deletedFileIds: [], failedFileIds: [...failed] };
+
+ let confirmed: string[];
+ try {
+ const result = await deleteFiles(pending, undefined, { files, userId });
+ if (!Array.isArray(result?.deletedFileIds) || !Array.isArray(result?.failedFileIds)) {
+ throw new Error('File metadata deletion was not confirmed');
+ }
+ const requested = new Set(pending);
+ const deleted = new Set(result.deletedFileIds);
+ for (const id of result.failedFileIds) {
+ if (!requested.has(id) || deleted.has(id)) throw new Error('Invalid file deletion outcome');
+ failed.add(id);
+ }
+ for (const id of deleted) {
+ if (!requested.has(id)) throw new Error('Invalid file deletion outcome');
+ }
+ confirmed = pending.filter((id) => deleted.has(id));
+ for (const id of pending) {
+ if (!deleted.has(id)) failed.add(id);
+ }
+ } catch (error) {
+ logger.error(
+ 'Error deleting file metadata after storage deletion',
+ getSafeErrorMetadata(error),
+ );
+ throw error;
+ }
+
+ if (confirmed.length > 0) {
+ try {
+ await removeAgentResourceFilesFromAllAgents({ file_ids: confirmed });
+ } catch (error) {
+ logger.error('Error cleaning up orphaned agent file references', getSafeErrorMetadata(error));
+ }
+ }
+ return { deletedFileIds: confirmed, failedFileIds: [...failed] };
+}
diff --git a/packages/api/src/files/index.ts b/packages/api/src/files/index.ts
index 6843fb56c6..f778b4bfa7 100644
--- a/packages/api/src/files/index.ts
+++ b/packages/api/src/files/index.ts
@@ -4,6 +4,10 @@ export * from './code';
export * from './citations';
export * from './context';
export * from './deletion';
+export * from './finalize';
+export * from './storage';
+export * from './operations';
+export * from './dispatch';
export * from './orphan';
export * from './extract';
export * from './documents/crud';
diff --git a/packages/api/src/files/operations.database.spec.ts b/packages/api/src/files/operations.database.spec.ts
new file mode 100644
index 0000000000..ae3c2df1af
--- /dev/null
+++ b/packages/api/src/files/operations.database.spec.ts
@@ -0,0 +1,313 @@
+import mongoose from 'mongoose';
+import { MongoMemoryServer } from 'mongodb-memory-server';
+import { FileContext, FileSources } from 'librechat-data-provider';
+import { createMethods, createModels } from '@librechat/data-schemas';
+import type { CodeFileCommitData, FileProvisionCommit, IMongoFile } from '@librechat/data-schemas';
+import { createFileProvisionRecovery, withFileProvisioning, withFileDeletion } from './operations';
+
+let mongo: MongoMemoryServer;
+let File: mongoose.Model<IMongoFile>;
+let methods: ReturnType<typeof createMethods>;
+const owner = new mongoose.Types.ObjectId();
+const scope = { file_id: 'recovery-file', userId: owner.toString(), tenantId: 'tenant' };
+const file = { file_id: scope.file_id, user: owner, tenantId: scope.tenantId };
+const ref = {
+ kind: 'user' as const,
+ id: scope.userId,
+ storage_session_id: 'session',
+ file_id: 'remote',
+};
+
+beforeAll(async () => {
+ mongo = await MongoMemoryServer.create();
+ createModels(mongoose);
+ await mongoose.connect(mongo.getUri());
+ File = mongoose.models.File as mongoose.Model<IMongoFile>;
+ methods = createMethods(mongoose);
+}, 30_000);
+
+afterAll(async () => {
+ await mongoose.disconnect();
+ await mongo.stop();
+});
+
+beforeEach(async () => {
+ await File.deleteMany({});
+ await File.create({
+ ...file,
+ filename: 'notes.txt',
+ filepath: '/uploads/notes.txt',
+ type: 'text/plain',
+ bytes: 1,
+ source: 'local',
+ });
+});
+
+it('recovers the original remote result and claim after the database becomes available', async () => {
+ const storage = { ...methods, recovery: createFileProvisionRecovery() };
+ const commit = jest
+ .spyOn(storage, 'commitFileProvision')
+ .mockRejectedValueOnce(new Error('database unavailable'))
+ .mockRejectedValueOnce(new Error('database unavailable'));
+ const upload = jest.fn(async () => ({
+ value: ref,
+ commit: { code: { routeKey: 'default', ref } },
+ }));
+ const options = {
+ file,
+ userId: scope.userId,
+ storage,
+ destination: 'code:default:notes.txt',
+ operation: upload,
+ };
+ await expect(withFileProvisioning(options)).rejects.toThrow('persistence was not confirmed');
+ const pending = await File.findOne({ file_id: file.file_id }).lean();
+ const token = pending?.storageOperations?.provisioning?.[0];
+ expect(token).toEqual(expect.any(String));
+
+ expect(await withFileProvisioning({ ...options, storage: { ...storage } })).toBe(ref);
+ expect(upload).toHaveBeenCalledTimes(1);
+ expect(commit).toHaveBeenCalledTimes(3);
+ expect(commit).toHaveBeenLastCalledWith(
+ expect.objectContaining({ claim: expect.objectContaining({ token }) }),
+ );
+ const stored = await File.findOne({ file_id: file.file_id }).lean();
+ expect(stored?.metadata?.codeEnvRefs?.default).toEqual(ref);
+ expect(stored?.storageOperations?.provisioning).toEqual([]);
+ expect(
+ await methods.acquireFileDeletion({
+ ...scope,
+ updatedAt: stored?.updatedAt,
+ token: 'deletion',
+ }),
+ ).toEqual({ resumed: false });
+});
+
+it('reconciles a lost acquisition acknowledgment without abandoning its applied token', async () => {
+ const storage = { ...methods, recovery: createFileProvisionRecovery() };
+ const acquire = jest
+ .spyOn(storage, 'acquireFileProvision')
+ .mockImplementationOnce(async (claim) => {
+ await methods.acquireFileProvision(claim);
+ throw new Error('acquisition acknowledgment lost');
+ });
+ const release = jest.spyOn(storage, 'releaseFileProvision');
+ const operation = jest.fn(async () => ({
+ value: ref,
+ commit: { code: { routeKey: 'default', ref } },
+ }));
+ const options = { file, userId: scope.userId, storage, destination: 'code', operation };
+ await expect(withFileProvisioning(options)).rejects.toThrow();
+ expect(operation).not.toHaveBeenCalled();
+ const original = acquire.mock.calls[0][0];
+ expect(
+ (await File.findOne({ file_id: file.file_id }).lean())?.storageOperations?.provisioning,
+ ).toEqual([original.token]);
+ await expect(withFileProvisioning(options)).resolves.toBe(ref);
+ expect(operation).toHaveBeenCalledTimes(1);
+ expect(release).toHaveBeenCalledWith(original);
+ const stored = await File.findOne({ file_id: file.file_id }).lean();
+ expect(stored?.storageOperations?.provisioning).toEqual([]);
+ expect(
+ await methods.acquireFileDeletion({ ...scope, updatedAt: stored?.updatedAt, token: 'delete' }),
+ ).toEqual({ resumed: false });
+});
+
+it('retries release only after a lost acknowledgment without overwriting a later pointer', async () => {
+ const storage = { ...methods, recovery: createFileProvisionRecovery() };
+ const release = jest
+ .spyOn(storage, 'releaseFileProvision')
+ .mockImplementationOnce(async (claim) => {
+ await methods.releaseFileProvision(claim);
+ throw new Error('release acknowledgment lost');
+ });
+ const commit = jest.spyOn(storage, 'commitFileProvision');
+ const operation = jest.fn(async () => ({
+ value: ref,
+ commit: { code: { routeKey: 'default', ref } },
+ }));
+ const options = {
+ file,
+ userId: scope.userId,
+ storage,
+ destination: 'code:default:notes.txt',
+ operation,
+ };
+ await expect(withFileProvisioning(options)).rejects.toThrow('persistence was not confirmed');
+
+ const peer = { ...scope, token: 'peer' };
+ const replacement: FileProvisionCommit = {
+ claim: peer,
+ code: { routeKey: 'default', ref: { ...ref, file_id: 'replacement' } },
+ };
+ expect(await methods.acquireFileProvision(peer)).toBe(true);
+ expect(await methods.commitFileProvision(replacement)).toBe(true);
+ expect(await methods.releaseFileProvision(peer)).toBe(true);
+ expect(await withFileProvisioning(options)).toBe(ref);
+ expect(operation).toHaveBeenCalledTimes(1);
+ expect(commit).toHaveBeenCalledTimes(1);
+ expect(release).toHaveBeenCalledTimes(2);
+ const stored = await File.findOne({ file_id: file.file_id }).lean();
+ expect(stored?.metadata?.codeEnvRefs?.default?.file_id).toBe('replacement');
+ expect(stored?.storageOperations?.provisioning).toEqual([]);
+});
+
+it.each([true, false])(
+ 'recovers a deletion acquisition with lost acknowledgement, applied=%s',
+ async (applied) => {
+ const storage = { ...methods, recovery: createFileProvisionRecovery() };
+ const stored = await File.findOne({ file_id: file.file_id }).lean();
+ const snapshot = { ...file, updatedAt: stored?.updatedAt };
+ const acquire = jest
+ .spyOn(storage, 'acquireFileDeletion')
+ .mockImplementationOnce(async (claim) => {
+ if (applied) expect(await methods.acquireFileDeletion(claim)).not.toBeNull();
+ throw new Error('deletion acquisition acknowledgement lost');
+ });
+ const release = jest.spyOn(storage, 'releaseFileDeletion');
+ const stale = jest.fn();
+ const options = { file: snapshot, userId: scope.userId, storage };
+ await expect(withFileDeletion({ ...options, operation: stale })).rejects.toThrow(
+ 'acknowledgement lost',
+ );
+ const original = acquire.mock.calls[0][0];
+ const operation = jest.fn(async (removed: () => void) => {
+ removed();
+ return ref;
+ });
+ await expect(withFileDeletion({ ...options, operation })).resolves.toBe(ref);
+ expect(stale).not.toHaveBeenCalled();
+ expect(operation).toHaveBeenCalledTimes(1);
+ expect(release.mock.calls[0][0]).toEqual({ ...original, retainDeletion: true });
+ const after = await File.findOne({ file_id: file.file_id }).lean();
+ expect(after?.storageOperations?.deleting).toBeUndefined();
+ expect(after?.storageOperations?.deletionPending).toBe(true);
+ expect(await methods.acquireFileProvision({ ...scope, token: 'upload' })).toBe(false);
+ },
+);
+
+it('retries only deletion release after its applied acknowledgement is lost', async () => {
+ const storage = { ...methods, recovery: createFileProvisionRecovery() };
+ const stored = await File.findOne({ file_id: file.file_id }).lean();
+ const release = jest
+ .spyOn(storage, 'releaseFileDeletion')
+ .mockImplementationOnce(async (claim) => {
+ expect(await methods.releaseFileDeletion(claim)).toBe(true);
+ throw new Error('deletion release acknowledgement lost');
+ });
+ const operation = jest.fn(async (removed: () => void) => {
+ removed();
+ return ref;
+ });
+ const options = {
+ file: { ...file, updatedAt: stored?.updatedAt },
+ userId: scope.userId,
+ storage,
+ operation,
+ };
+ await expect(withFileDeletion(options)).rejects.toThrow('acknowledgement lost');
+ const replacement = jest.fn();
+ await expect(withFileDeletion({ ...options, operation: replacement })).resolves.toBe(ref);
+ expect(operation).toHaveBeenCalledTimes(1);
+ expect(replacement).not.toHaveBeenCalled();
+ expect(release).toHaveBeenCalledTimes(2);
+ expect(release.mock.calls[0][0]).toEqual(release.mock.calls[1][0]);
+ const after = await File.findOne({ file_id: file.file_id }).lean();
+ expect(after?.storageOperations?.deleting).toBeUndefined();
+ expect(after?.storageOperations?.deletionPending).toBe(true);
+ expect(await methods.acquireFileProvision({ ...scope, token: 'upload' })).toBe(false);
+});
+
+it('never clears a newer deletion token or its pending intent when retrying release', async () => {
+ const storage = { ...methods, recovery: createFileProvisionRecovery() };
+ const stored = await File.findOne({ file_id: file.file_id }).lean();
+ jest.spyOn(storage, 'releaseFileDeletion').mockImplementationOnce(async (claim) => {
+ expect(await methods.releaseFileDeletion(claim)).toBe(true);
+ throw new Error('acknowledgement lost');
+ });
+ const options = {
+ file: { ...file, updatedAt: stored?.updatedAt },
+ userId: scope.userId,
+ storage,
+ };
+ await expect(withFileDeletion({ ...options, operation: async () => ref })).rejects.toThrow();
+ const peer = { ...scope, updatedAt: stored?.updatedAt, token: 'foreign' };
+ expect(await methods.acquireFileDeletion(peer)).toEqual({ resumed: false });
+ const replacement = jest.fn();
+ await expect(withFileDeletion({ ...options, operation: replacement })).rejects.toThrow(
+ 'could not be released',
+ );
+ expect(replacement).not.toHaveBeenCalled();
+ const after = await File.findOne({ file_id: file.file_id }).lean();
+ expect(after?.storageOperations?.deleting).toBe('foreign');
+ expect(after?.storageOperations?.deletionPending).toBe(true);
+});
+
+it('keeps regenerated output writes fenced until their complete record is committed', async () => {
+ const storage = { ...methods, recovery: createFileProvisionRecovery() };
+ const output: CodeFileCommitData = {
+ file_id: file.file_id,
+ user: scope.userId,
+ tenantId: scope.tenantId,
+ filename: 'notes.txt',
+ filepath: '/uploads/notes.txt',
+ type: 'text/plain',
+ bytes: 8,
+ object: 'file',
+ usage: 2,
+ source: FileSources.local,
+ context: FileContext.execute_code,
+ conversationId: 'conversation',
+ metadata: { codeEnvRefs: { default: ref } },
+ };
+ const saved = await withFileProvisioning({
+ file,
+ userId: scope.userId,
+ storage,
+ destination: 'generated-output',
+ operation: async () => {
+ const before = await File.findOne({ file_id: file.file_id }).lean();
+ expect(
+ await methods.acquireFileDeletion({
+ ...scope,
+ updatedAt: before?.updatedAt,
+ token: 'blocked',
+ }),
+ ).toBeNull();
+ return { value: output, commit: { output: { file: output } } };
+ },
+ });
+ expect(saved).toBe(output);
+ const row = await File.findOne({ file_id: file.file_id }).lean();
+ expect(row?.usage).toBe(2);
+ expect(row?.metadata?.codeEnvRefs?.default).toEqual(ref);
+ expect(row?.storageOperations?.provisioning).toEqual([]);
+ expect(
+ await methods.acquireFileDeletion({ ...scope, updatedAt: row?.updatedAt, token: 'next' }),
+ ).toEqual({ resumed: false });
+});
+
+it('does not overwrite fresh output bytes while deletion intent remains after primary removal', async () => {
+ const storage = { ...methods, recovery: createFileProvisionRecovery() };
+ const row = await File.findOne({ file_id: file.file_id }).lean();
+ expect(
+ await methods.acquireFileDeletion({ ...scope, updatedAt: row?.updatedAt, token: 'delete' }),
+ ).toEqual({ resumed: false });
+ expect(
+ await methods.releaseFileDeletion({ ...scope, token: 'delete', retainDeletion: true }),
+ ).toBe(true);
+ const writePrimary = jest.fn();
+ await expect(
+ withFileProvisioning({
+ file,
+ userId: scope.userId,
+ storage,
+ destination: 'generated-output',
+ operation: writePrimary,
+ }),
+ ).rejects.toThrow('conflicts');
+ expect(writePrimary).not.toHaveBeenCalled();
+ expect(
+ (await File.findOne({ file_id: file.file_id }).lean())?.storageOperations?.deletionPending,
+ ).toBe(true);
+});
diff --git a/packages/api/src/files/operations.spec.ts b/packages/api/src/files/operations.spec.ts
new file mode 100644
index 0000000000..d63491475b
--- /dev/null
+++ b/packages/api/src/files/operations.spec.ts
@@ -0,0 +1,1065 @@
+import type { FileStorageClaim, FileProvisionCommit } from '@librechat/data-schemas';
+import type { FileDeleteStorage } from './operations';
+import {
+ withFileDeletion,
+ withFileProvisioning,
+ confirmFileProvision,
+ createFileProvisionRecovery,
+ FileProvisionPersistenceError,
+ FileProvisionOutcomeUnknownError,
+ isDefinitiveFileStorageFailure,
+} from './operations';
+
+const file = { file_id: 'file', user: 'owner', tenantId: 'tenant' };
+const scope = { file_id: file.file_id, userId: 'owner', tenantId: file.tenantId };
+const result = (value = 'uploaded') => ({
+ value,
+ commit: { embedded: true, entityId: 'original' },
+});
+const key = (claim: FileStorageClaim) =>
+ JSON.stringify([claim.tenantId ?? null, claim.userId, claim.file_id]);
+
+function storageStatusErrors(status: unknown) {
+ return [
+ { response: { status } },
+ { $metadata: { httpStatusCode: status }, code: 'AccessDenied' },
+ { statusCode: status },
+ { status },
+ ];
+}
+
+function setup() {
+ const provisioners = new Map<string, string>();
+ const deleting = new Map<string, string>();
+ const pending = new Set<string>();
+ const storage = {
+ recovery: createFileProvisionRecovery(),
+ commitFileProvision: jest.fn(async (_data: FileProvisionCommit): Promise<boolean> => true),
+ acquireFileProvision: jest.fn(async (claim: FileStorageClaim) => {
+ if (pending.has(key(claim)) || deleting.has(key(claim))) return false;
+ provisioners.set(claim.token, key(claim));
+ return true;
+ }),
+ releaseFileProvision: jest.fn(async (claim: FileStorageClaim): Promise<boolean> => {
+ if (provisioners.get(claim.token) === key(claim)) provisioners.delete(claim.token);
+ return true;
+ }),
+ acquireFileDeletion: jest.fn(async (claim: FileStorageClaim) => {
+ if (deleting.has(key(claim)) || [...provisioners.values()].includes(key(claim))) return null;
+ const resumed = pending.has(key(claim));
+ deleting.set(key(claim), claim.token);
+ pending.add(key(claim));
+ return { resumed };
+ }),
+ releaseFileDeletion: jest.fn(async (claim: FileStorageClaim & { retainDeletion: boolean }) => {
+ if (!deleting.has(key(claim))) return true;
+ if (claim.token !== deleting.get(key(claim))) return false;
+ deleting.delete(key(claim));
+ if (!claim.retainDeletion) pending.delete(key(claim));
+ return true;
+ }),
+ } satisfies FileDeleteStorage;
+ return { storage, provisioners, deleting, pending };
+}
+
+function provision(
+ storage: FileDeleteStorage,
+ operation: (
+ claim: FileStorageClaim,
+ onUploadStart: () => void,
+ ) => Promise<ReturnType<typeof result>> = async () => result(),
+ destination = 'code:route:sandbox',
+ identity = file,
+) {
+ return withFileProvisioning({
+ file: identity,
+ userId: 'caller',
+ destination,
+ storage,
+ operation,
+ });
+}
+
+function barrier() {
+ let release: () => void = () => undefined;
+ const wait = new Promise<void>((resolve) => {
+ release = resolve;
+ });
+ return { wait, release };
+}
+
+afterEach(() => jest.useRealTimers());
+
+it('blocks provisioning throughout primary I/O and until metadata finalizes', async () => {
+ const { storage } = setup();
+ const entered = barrier();
+ const complete = barrier();
+ const deletion = withFileDeletion({
+ file,
+ userId: 'caller',
+ storage,
+ operation: async (primaryDeleted) => {
+ entered.release();
+ await complete.wait;
+ primaryDeleted();
+ },
+ });
+ await entered.wait;
+ const upload = jest.fn(async () => result());
+ await expect(provision(storage, upload)).rejects.toThrow('conflicts');
+ complete.release();
+ await deletion;
+ await expect(provision(storage, upload)).rejects.toThrow('conflicts');
+ expect(upload).not.toHaveBeenCalled();
+});
+
+it('denies deletion immediately during an active upload without waiting for it', async () => {
+ const { storage } = setup();
+ const entered = barrier();
+ const done = barrier();
+ const upload = jest.fn(async () => {
+ entered.release();
+ await done.wait;
+ return result();
+ });
+ const work = provision(storage, upload);
+ await entered.wait;
+ const remove = jest.fn();
+ await expect(
+ withFileDeletion({ file, userId: 'caller', storage, operation: remove }),
+ ).rejects.toThrow('conflicts');
+ expect(remove).not.toHaveBeenCalled();
+ done.release();
+ await work;
+ await withFileDeletion({ file, userId: 'caller', storage, operation: remove });
+ expect(remove).toHaveBeenCalledTimes(1);
+});
+
+it('restores repair eligibility after deletion fails before primary removal', async () => {
+ const { storage } = setup();
+ await expect(
+ withFileDeletion({
+ file,
+ userId: 'caller',
+ storage,
+ operation: async () => {
+ throw new Error('secondary offline');
+ },
+ }),
+ ).rejects.toThrow('secondary offline');
+ await expect(provision(storage)).resolves.toBe('uploaded');
+});
+
+it('retains deletion intent after a failed retry of an already-removed primary', async () => {
+ const { storage } = setup();
+ await withFileDeletion({
+ file,
+ userId: 'caller',
+ storage,
+ operation: async (removed) => removed(),
+ });
+ await expect(
+ withFileDeletion({
+ file,
+ userId: 'caller',
+ storage,
+ operation: async () => {
+ throw new Error('metadata offline');
+ },
+ }),
+ ).rejects.toThrow('metadata offline');
+ await expect(provision(storage)).rejects.toThrow('conflicts');
+});
+
+it('releases the original claim after a failed upload before deletion', async () => {
+ const { storage } = setup();
+ const upload = jest.fn(async () => {
+ throw new Error('upload offline');
+ });
+ await expect(provision(storage, upload)).rejects.toThrow('upload offline');
+ expect(storage.releaseFileProvision).toHaveBeenCalledWith(
+ storage.acquireFileProvision.mock.calls[0][0],
+ );
+ expect(storage.commitFileProvision).not.toHaveBeenCalled();
+ await withFileDeletion({ file, userId: 'caller', storage, operation: jest.fn() });
+});
+
+it.each([true, false])(
+ 'reconciles the original acquisition token before a fresh upload when applied=%s',
+ async (applied) => {
+ const { storage, provisioners } = setup();
+ const failure = new Error('private acquisition acknowledgement failure');
+ storage.acquireFileProvision.mockImplementationOnce(async (claim) => {
+ if (applied) provisioners.set(claim.token, key(claim));
+ throw failure;
+ });
+ const originalUpload = jest.fn(async () => result('stale request'));
+ await expect(provision(storage, originalUpload)).rejects.toMatchObject({
+ name: 'FileProvisionPersistenceError',
+ message: 'File provisioning persistence was not confirmed',
+ cause: failure,
+ });
+ const original = storage.acquireFileProvision.mock.calls[0][0];
+ expect(storage.releaseFileProvision).not.toHaveBeenCalled();
+ const currentUpload = jest.fn(async (claim: FileStorageClaim) => {
+ expect(storage.releaseFileProvision).toHaveBeenLastCalledWith(original);
+ expect(provisioners.has(original.token)).toBe(false);
+ expect(claim.token).not.toBe(original.token);
+ return result('current request');
+ });
+ await expect(provision({ ...storage }, currentUpload)).resolves.toBe('current request');
+ expect(originalUpload).not.toHaveBeenCalled();
+ expect(currentUpload).toHaveBeenCalledTimes(1);
+ expect(storage.acquireFileProvision).toHaveBeenCalledTimes(2);
+ expect(storage.releaseFileProvision.mock.invocationCallOrder[0]).toBeLessThan(
+ storage.acquireFileProvision.mock.invocationCallOrder[1],
+ );
+ expect(provisioners.size).toBe(0);
+ await withFileDeletion({ file, userId: 'caller', storage, operation: jest.fn() });
+ },
+);
+
+it.each([true, false])(
+ 'releases a settled failed acquisition before deletion when applied=%s',
+ async (applied) => {
+ const { storage, provisioners } = setup();
+ storage.acquireFileProvision.mockImplementationOnce(async (claim) => {
+ if (applied) provisioners.set(claim.token, key(claim));
+ throw new Error('acknowledgement lost');
+ });
+ const upload = jest.fn(async () => result());
+ await expect(provision(storage, upload)).rejects.toBeInstanceOf(FileProvisionPersistenceError);
+ const original = storage.acquireFileProvision.mock.calls[0][0];
+ await withFileDeletion({ file, userId: 'caller', storage, operation: jest.fn() });
+ expect(storage.releaseFileProvision).toHaveBeenCalledWith(original);
+ expect(storage.releaseFileProvision.mock.invocationCallOrder[0]).toBeLessThan(
+ storage.acquireFileDeletion.mock.invocationCallOrder[0],
+ );
+ expect(provisioners.size).toBe(0);
+ expect(upload).not.toHaveBeenCalled();
+ expect(storage.commitFileProvision).not.toHaveBeenCalled();
+ },
+);
+
+it('keeps failed acquisition recovery fenced until the original token can be released', async () => {
+ const { storage, provisioners } = setup();
+ storage.acquireFileProvision.mockImplementationOnce(async (claim) => {
+ provisioners.set(claim.token, key(claim));
+ throw new Error('acknowledgement lost');
+ });
+ await expect(provision(storage)).rejects.toBeInstanceOf(FileProvisionPersistenceError);
+ const original = storage.acquireFileProvision.mock.calls[0][0];
+ storage.releaseFileProvision
+ .mockRejectedValueOnce(new Error('database offline'))
+ .mockResolvedValueOnce(false);
+ const upload = jest.fn(async () => result());
+ await expect(provision(storage, upload)).rejects.toBeInstanceOf(FileProvisionPersistenceError);
+ await expect(
+ withFileDeletion({ file, userId: 'caller', storage, operation: jest.fn() }),
+ ).rejects.toBeInstanceOf(FileProvisionPersistenceError);
+ expect(storage.acquireFileProvision).toHaveBeenCalledTimes(1);
+ expect(storage.acquireFileDeletion).not.toHaveBeenCalled();
+ expect(upload).not.toHaveBeenCalled();
+ expect(provisioners.has(original.token)).toBe(true);
+ await expect(provision(storage, upload)).resolves.toBe('uploaded');
+ expect(provisioners.has(original.token)).toBe(false);
+});
+
+it('does not wait for or release a pending acquisition during deletion recovery', async () => {
+ const { storage, provisioners } = setup();
+ const entered = barrier();
+ const done = barrier();
+ const failure = new Error('acknowledgement lost');
+ storage.acquireFileProvision.mockImplementationOnce(async (claim) => {
+ provisioners.set(claim.token, key(claim));
+ entered.release();
+ await done.wait;
+ throw failure;
+ });
+ const upload = jest.fn(async () => result());
+ const work = provision(storage, upload);
+ const rejected = work.catch((error: unknown) => error);
+ await entered.wait;
+ await storage.recovery.recover(scope, storage);
+ await expect(
+ withFileDeletion({ file, userId: 'caller', storage, operation: jest.fn() }),
+ ).rejects.toThrow('conflicts');
+ expect(storage.releaseFileProvision).not.toHaveBeenCalled();
+ expect(upload).not.toHaveBeenCalled();
+ done.release();
+ expect(await rejected).toMatchObject({ cause: failure });
+ await storage.recovery.recover(scope, storage);
+ expect(storage.releaseFileProvision).toHaveBeenCalledWith(
+ storage.acquireFileProvision.mock.calls[0][0],
+ );
+ expect(provisioners.size).toBe(0);
+});
+
+it('retains a synchronous acquisition failure without reentrant recovery releasing it', async () => {
+ const { storage } = setup();
+ const failure = new Error('synchronous acquisition failure');
+ let recovery: Promise<void> | undefined;
+ storage.acquireFileProvision.mockImplementationOnce(() => {
+ recovery = storage.recovery.recover(scope, storage);
+ throw failure;
+ });
+ await expect(provision(storage)).rejects.toMatchObject({ cause: failure });
+ await recovery;
+ expect(storage.releaseFileProvision).not.toHaveBeenCalled();
+ await expect(provision(storage)).resolves.toBe('uploaded');
+ expect(storage.acquireFileProvision).toHaveBeenCalledTimes(2);
+});
+
+it('assigns the recovery attempt before a release can reenter recovery', async () => {
+ const { storage, provisioners } = setup();
+ storage.acquireFileProvision.mockRejectedValueOnce(new Error('acknowledgement lost'));
+ await expect(provision(storage)).rejects.toBeInstanceOf(FileProvisionPersistenceError);
+ let reentrant: Promise<void> | undefined;
+ storage.releaseFileProvision.mockImplementationOnce(async (claim) => {
+ reentrant = storage.recovery.recover(scope, storage);
+ provisioners.delete(claim.token);
+ return true;
+ });
+ await storage.recovery.recover(scope, storage);
+ await reentrant;
+ expect(storage.releaseFileProvision).toHaveBeenCalledTimes(1);
+ expect(storage.commitFileProvision).not.toHaveBeenCalled();
+});
+
+it('discards an explicitly contended acquisition without attempting to release it', async () => {
+ const { storage } = setup();
+ storage.acquireFileProvision.mockResolvedValueOnce(false);
+ await expect(provision(storage)).rejects.toThrow('conflicts');
+ expect(storage.releaseFileProvision).not.toHaveBeenCalled();
+ await expect(provision(storage)).resolves.toBe('uploaded');
+});
+
+it('reuses one active upload for overlapping same-destination calls', async () => {
+ const { storage } = setup();
+ const entered = barrier();
+ const done = barrier();
+ const upload = jest.fn(async () => {
+ entered.release();
+ await done.wait;
+ return result();
+ });
+ const first = provision(storage, upload);
+ await entered.wait;
+ const secondUpload = jest.fn(async () => result('wrong'));
+ const second = provision(storage, secondUpload);
+ done.release();
+ await expect(Promise.all([first, second])).resolves.toEqual(['uploaded', 'uploaded']);
+ expect(upload).toHaveBeenCalledTimes(1);
+ expect(secondUpload).not.toHaveBeenCalled();
+ expect(storage.commitFileProvision).toHaveBeenCalledTimes(1);
+});
+
+it('recovers two failed commit attempts with the original token and no new upload', async () => {
+ const { storage, provisioners } = setup();
+ storage.commitFileProvision
+ .mockResolvedValueOnce(false)
+ .mockRejectedValueOnce(new Error('database'));
+ const uploaded = result();
+ const upload = jest.fn(async () => uploaded);
+ await expect(provision(storage, upload)).rejects.toBeInstanceOf(FileProvisionPersistenceError);
+ expect(storage.commitFileProvision).toHaveBeenCalledTimes(2);
+ expect(storage.releaseFileProvision).not.toHaveBeenCalled();
+ expect(provisioners.size).toBe(1);
+ uploaded.commit.entityId = 'mutated';
+ const freshCallback = jest.fn(async () => result('wrong'));
+ await expect(provision({ ...storage }, freshCallback)).resolves.toBe('uploaded');
+ expect(upload).toHaveBeenCalledTimes(1);
+ expect(freshCallback).not.toHaveBeenCalled();
+ expect(storage.acquireFileProvision).toHaveBeenCalledTimes(1);
+ const original = storage.acquireFileProvision.mock.calls[0][0];
+ for (const [commit] of storage.commitFileProvision.mock.calls) {
+ expect(commit.claim).toEqual(original);
+ expect(commit.entityId).toBe('original');
+ }
+ expect(storage.releaseFileProvision).toHaveBeenCalledWith(original);
+ expect(provisioners.size).toBe(0);
+});
+
+it('serializes two overlapping recovery calls into one commit', async () => {
+ const { storage } = setup();
+ storage.commitFileProvision.mockResolvedValueOnce(false).mockResolvedValueOnce(false);
+ await expect(provision(storage)).rejects.toBeInstanceOf(FileProvisionPersistenceError);
+ const entered = barrier();
+ const done = barrier();
+ storage.commitFileProvision.mockImplementationOnce(async () => {
+ entered.release();
+ await done.wait;
+ return true;
+ });
+ const first = storage.recovery.recover(scope, storage);
+ await entered.wait;
+ const second = storage.recovery.recover(scope, storage);
+ done.release();
+ await Promise.all([first, second]);
+ expect(storage.commitFileProvision).toHaveBeenCalledTimes(3);
+ expect(storage.releaseFileProvision).toHaveBeenCalledTimes(1);
+});
+
+it('retries only release after a confirmed commit without overwriting its result', async () => {
+ const { storage } = setup();
+ storage.releaseFileProvision.mockResolvedValueOnce(false);
+ const upload = jest.fn(async () => result());
+ await expect(provision(storage, upload)).rejects.toBeInstanceOf(FileProvisionPersistenceError);
+ const replacement = jest.fn(async () => result('wrong'));
+ await expect(provision(storage, replacement)).resolves.toBe('uploaded');
+ expect(replacement).not.toHaveBeenCalled();
+ expect(storage.commitFileProvision).toHaveBeenCalledTimes(1);
+ expect(storage.releaseFileProvision).toHaveBeenCalledTimes(2);
+ expect(storage.releaseFileProvision.mock.calls[0][0]).toEqual(
+ storage.releaseFileProvision.mock.calls[1][0],
+ );
+});
+
+it('finishes release-only upload failure recovery before acquiring a new token', async () => {
+ const { storage } = setup();
+ storage.releaseFileProvision.mockRejectedValueOnce(new Error('release offline'));
+ const failure = new Error('original upload failure');
+ await expect(
+ provision(
+ storage,
+ jest.fn(async () => {
+ throw failure;
+ }),
+ ),
+ ).rejects.toBe(failure);
+ const original = storage.acquireFileProvision.mock.calls[0][0];
+ const upload = jest.fn(async () => {
+ expect(storage.releaseFileProvision).toHaveBeenLastCalledWith(original);
+ return result('new upload');
+ });
+ await expect(provision(storage, upload)).resolves.toBe('new upload');
+ expect(storage.acquireFileProvision).toHaveBeenCalledTimes(2);
+ expect(storage.acquireFileProvision.mock.calls[1][0].token).not.toBe(original.token);
+ expect(storage.commitFileProvision).toHaveBeenCalledTimes(1);
+});
+
+it('keeps release-only recovery when releasing a failed upload still fails', async () => {
+ const { storage } = setup();
+ storage.releaseFileProvision.mockResolvedValue(false);
+ await expect(
+ provision(
+ storage,
+ jest.fn(async () => {
+ throw new Error('upload');
+ }),
+ ),
+ ).rejects.toThrow('upload');
+ const next = jest.fn(async () => result());
+ await expect(provision(storage, next)).rejects.toBeInstanceOf(FileProvisionPersistenceError);
+ expect(next).not.toHaveBeenCalled();
+ expect(storage.acquireFileProvision).toHaveBeenCalledTimes(1);
+ storage.releaseFileProvision.mockResolvedValue(true);
+ await expect(provision(storage, next)).resolves.toBe('uploaded');
+});
+
+it('recovers a completed upload before acquiring deletion', async () => {
+ const { storage, provisioners } = setup();
+ storage.commitFileProvision.mockResolvedValueOnce(false).mockResolvedValueOnce(false);
+ await expect(provision(storage)).rejects.toBeInstanceOf(FileProvisionPersistenceError);
+ const remove = jest.fn(async () => {
+ expect(provisioners.size).toBe(0);
+ expect(storage.commitFileProvision).toHaveBeenCalledTimes(3);
+ });
+ await withFileDeletion({ file, userId: 'caller', storage, operation: remove });
+ expect(remove).toHaveBeenCalledTimes(1);
+});
+
+it('does not acquire deletion when known recovery still fails', async () => {
+ const { storage } = setup();
+ storage.commitFileProvision.mockResolvedValue(false);
+ await expect(provision(storage)).rejects.toBeInstanceOf(FileProvisionPersistenceError);
+ await expect(
+ withFileDeletion({ file, userId: 'caller', storage, operation: jest.fn() }),
+ ).rejects.toBeInstanceOf(FileProvisionPersistenceError);
+ expect(storage.acquireFileDeletion).not.toHaveBeenCalled();
+ expect(storage.releaseFileProvision).not.toHaveBeenCalled();
+});
+
+it.each([
+ { ...file, file_id: 'other-file' },
+ { ...file, user: 'other-owner' },
+ { ...file, tenantId: 'other-tenant' },
+])('isolates failed recovery from scope %j', async (other) => {
+ const { storage } = setup();
+ storage.commitFileProvision.mockResolvedValueOnce(false).mockResolvedValueOnce(false);
+ await expect(provision(storage)).rejects.toBeInstanceOf(FileProvisionPersistenceError);
+ const otherUpload = jest.fn(async () => result('other'));
+ await expect(provision(storage, otherUpload, 'code:route:sandbox', other)).resolves.toBe('other');
+ expect(otherUpload).toHaveBeenCalledTimes(1);
+ await storage.recovery.recover(scope, storage);
+});
+
+it('keeps destinations independent while deletion waits for every claim', async () => {
+ const { storage, provisioners } = setup();
+ const entered = barrier();
+ const done = barrier();
+ let started = 0;
+ const upload = jest.fn(async () => {
+ if (++started === 2) entered.release();
+ await done.wait;
+ return result();
+ });
+ const uploads = [provision(storage, upload, 'route:a'), provision(storage, upload, 'route:b')];
+ await entered.wait;
+ expect(provisioners.size).toBe(2);
+ await expect(
+ withFileDeletion({ file, userId: 'caller', storage, operation: jest.fn() }),
+ ).rejects.toThrow('conflicts');
+ done.release();
+ await Promise.all(uploads);
+ expect(provisioners.size).toBe(0);
+});
+
+it('never releases an unknown foreign claim', async () => {
+ const { storage, provisioners } = setup();
+ const foreign = { ...scope, token: 'foreign-worker' };
+ await storage.acquireFileProvision(foreign);
+ await storage.recovery.recover(scope, storage);
+ await expect(
+ withFileDeletion({ file, userId: 'caller', storage, operation: jest.fn() }),
+ ).rejects.toThrow('conflicts');
+ expect(storage.releaseFileProvision).not.toHaveBeenCalled();
+ expect(provisioners.has(foreign.token)).toBe(true);
+});
+
+it('does not expire a known remote result', async () => {
+ const { storage } = setup();
+ storage.commitFileProvision.mockResolvedValueOnce(false).mockResolvedValueOnce(false);
+ const upload = jest.fn(async () => result());
+ await expect(provision(storage, upload)).rejects.toBeInstanceOf(FileProvisionPersistenceError);
+ jest.useFakeTimers();
+ jest.setSystemTime(new Date('2099-01-01'));
+ await expect(provision(storage, upload)).resolves.toBe('uploaded');
+ expect(upload).toHaveBeenCalledTimes(1);
+});
+
+it('removes successful entries so a later independent call uploads again', async () => {
+ const { storage } = setup();
+ const upload = jest.fn(async () => result());
+ await provision(storage, upload);
+ await provision(storage, upload);
+ expect(upload).toHaveBeenCalledTimes(2);
+ expect(storage.acquireFileProvision).toHaveBeenCalledTimes(2);
+});
+
+it('preserves generic value references during recovery', async () => {
+ const { storage } = setup();
+ storage.commitFileProvision.mockResolvedValueOnce(false).mockResolvedValueOnce(false);
+ const value = { remote: 'ref' };
+ const options = {
+ file,
+ userId: 'caller',
+ destination: 'vector:namespace',
+ storage,
+ operation: jest.fn(async () => ({ value, commit: { embedded: true } })),
+ };
+ await expect(withFileProvisioning(options)).rejects.toBeInstanceOf(FileProvisionPersistenceError);
+ await expect(withFileProvisioning(options)).resolves.toBe(value);
+ expect(options.operation).toHaveBeenCalledTimes(1);
+});
+
+it('retains the standalone safe persistence error contract', async () => {
+ const { storage } = setup();
+ storage.commitFileProvision.mockRejectedValue(new Error('private database detail'));
+ await expect(
+ confirmFileProvision(storage, { claim: { ...scope, token: 'token' } }),
+ ).rejects.toThrow('File provisioning persistence was not confirmed');
+});
+
+it('releases a preflight failure that never started the upload', async () => {
+ const { storage, provisioners } = setup();
+ const error = new Error('storage read failed');
+ const upload = jest.fn(async (_claim: FileStorageClaim, _start: () => void) => {
+ throw error;
+ });
+ await expect(provision(storage, upload)).rejects.toBe(error);
+ expect(provisioners.size).toBe(0);
+ expect(storage.releaseFileProvision).toHaveBeenCalledWith(
+ storage.acquireFileProvision.mock.calls[0][0],
+ );
+ await expect(provision(storage)).resolves.toBe('uploaded');
+});
+
+it.each([
+ Object.assign(new Error('timeout'), { code: 'ETIMEDOUT' }),
+ Object.assign(new Error('aborted'), { name: 'AbortError' }),
+ new Error('lost response'),
+ ...[408, 409, 500, 503].flatMap(storageStatusErrors),
+])('retains an ambiguous started upload without retrying or releasing: %j', async (error) => {
+ const { storage, provisioners } = setup();
+ const upload = jest.fn(async (_claim: FileStorageClaim, start: () => void) => {
+ start();
+ throw error;
+ });
+ await expect(provision(storage, upload)).rejects.toBeInstanceOf(FileProvisionOutcomeUnknownError);
+ expect(provisioners.size).toBe(1);
+ const replacement = jest.fn(async () => result('wrong'));
+ await expect(provision(storage, replacement)).rejects.toBeInstanceOf(
+ FileProvisionOutcomeUnknownError,
+ );
+ await expect(storage.recovery.recover(scope, storage)).rejects.toBeInstanceOf(
+ FileProvisionOutcomeUnknownError,
+ );
+ const remove = jest.fn();
+ await expect(
+ withFileDeletion({ file, userId: 'caller', storage, operation: remove }),
+ ).rejects.toBeInstanceOf(FileProvisionOutcomeUnknownError);
+ expect(remove).not.toHaveBeenCalled();
+ expect(replacement).not.toHaveBeenCalled();
+ expect(upload).toHaveBeenCalledTimes(1);
+ expect(storage.acquireFileProvision).toHaveBeenCalledTimes(1);
+ expect(storage.commitFileProvision).not.toHaveBeenCalled();
+ expect(storage.releaseFileProvision).not.toHaveBeenCalled();
+ expect(storage.acquireFileDeletion).not.toHaveBeenCalled();
+});
+
+it.each([403, 429].flatMap(storageStatusErrors))(
+ 'releases a definitive SDK upload rejection and permits a fresh upload: %j',
+ async (error) => {
+ const { storage, provisioners } = setup();
+ const upload = jest.fn(async (_claim: FileStorageClaim, start: () => void) => {
+ start();
+ throw error;
+ });
+ await expect(provision(storage, upload)).rejects.toBe(error);
+ expect(provisioners.size).toBe(0);
+ expect(storage.releaseFileProvision).toHaveBeenCalledTimes(1);
+ const fresh = jest.fn(async () => result('fresh'));
+ await expect(provision(storage, fresh)).resolves.toBe('fresh');
+ expect(fresh).toHaveBeenCalledTimes(1);
+ expect(storage.acquireFileProvision).toHaveBeenCalledTimes(2);
+ },
+);
+
+it.each([
+ new Error('lost deletion response'),
+ Object.assign(new Error('aborted'), { name: 'AbortError' }),
+ ...[408, 409, 500, 503].flatMap(storageStatusErrors),
+])('retains deletion intent after ambiguous primary dispatch: %j', async (error) => {
+ const { storage } = setup();
+ await expect(
+ withFileDeletion({
+ file,
+ userId: 'caller',
+ storage,
+ operation: async (primaryDeleted) => {
+ primaryDeleted(false);
+ throw error;
+ },
+ }),
+ ).rejects.toBe(error);
+ expect(storage.releaseFileDeletion).toHaveBeenLastCalledWith(
+ expect.objectContaining({ retainDeletion: true }),
+ );
+ await expect(provision(storage)).rejects.toThrow('conflicts');
+});
+
+it.each([...storageStatusErrors(403), Object.assign(new Error('denied'), { code: 'EACCES' })])(
+ 'clears newly acquired deletion intent after a definitive primary rejection: %j',
+ async (error) => {
+ const { storage } = setup();
+ await expect(
+ withFileDeletion({
+ file,
+ userId: 'caller',
+ storage,
+ operation: async (primaryDeleted) => {
+ primaryDeleted(false);
+ throw error;
+ },
+ }),
+ ).rejects.toBe(error);
+ expect(storage.releaseFileDeletion).toHaveBeenLastCalledWith(
+ expect.objectContaining({ retainDeletion: false }),
+ );
+ await expect(provision(storage)).resolves.toBe('uploaded');
+ },
+);
+
+it.each(storageStatusErrors(403))(
+ 'never clears resumed deletion intent on an SDK rejection: %j',
+ async (error) => {
+ const { storage } = setup();
+ await withFileDeletion({
+ file,
+ userId: 'caller',
+ storage,
+ operation: async (primaryDeleted) => primaryDeleted(),
+ });
+ await expect(
+ withFileDeletion({
+ file,
+ userId: 'caller',
+ storage,
+ operation: async (primaryDeleted) => {
+ primaryDeleted(false);
+ throw error;
+ },
+ }),
+ ).rejects.toBe(error);
+ expect(storage.releaseFileDeletion).toHaveBeenLastCalledWith(
+ expect.objectContaining({ retainDeletion: true }),
+ );
+ await expect(provision(storage)).rejects.toThrow('conflicts');
+ },
+);
+
+it.each([400, 401, 403, 404, 413, 415, 422, 429].flatMap(storageStatusErrors))(
+ 'classifies explicit numeric SDK HTTP denial as definitive: %j',
+ (error) => {
+ expect(isDefinitiveFileStorageFailure(error)).toBe(true);
+ },
+);
+
+it.each(['EACCES', 'EPERM', 'CODE_API_RATE_LIMITED'])(
+ 'classifies safe pre-dispatch code %s as definitive',
+ (code) => {
+ expect(isDefinitiveFileStorageFailure({ code })).toBe(true);
+ },
+);
+
+it.each([
+ undefined,
+ null,
+ '403',
+ ...storageStatusErrors('403'),
+ ...storageStatusErrors(undefined),
+ ...storageStatusErrors(null),
+ { response: { status: 408 } },
+ { response: { status: 409 } },
+ { response: { status: 502 } },
+ { code: 'ECONNRESET' },
+ { name: 'AbortError' },
+])('does not infer definitive rejection from %j', (error) => {
+ expect(isDefinitiveFileStorageFailure(error)).toBe(false);
+});
+
+it.each([1, 2])('classifies a RAG HTTP denial through %s wrapped causes', async (depth) => {
+ const { storage } = setup();
+ let error: unknown = { response: { status: 403 } };
+ for (let index = 0; index < depth; index++)
+ error = new Error('RAG operation failed', { cause: error });
+ expect(isDefinitiveFileStorageFailure(error)).toBe(true);
+ const upload = jest.fn(async (_claim: FileStorageClaim, start: () => void) => {
+ start();
+ throw error;
+ });
+ await expect(provision(storage, upload)).rejects.toBe(error);
+ expect(storage.releaseFileProvision).toHaveBeenCalledTimes(1);
+ await expect(provision(storage)).resolves.toBe('uploaded');
+});
+
+it('retains an ambiguous wrapped RAG network failure', async () => {
+ const { storage } = setup();
+ const error = new Error('RAG operation failed', {
+ cause: Object.assign(new Error('timeout'), { code: 'ETIMEDOUT' }),
+ });
+ expect(isDefinitiveFileStorageFailure(error)).toBe(false);
+ const upload = jest.fn(async (_claim: FileStorageClaim, start: () => void) => {
+ start();
+ throw error;
+ });
+ await expect(provision(storage, upload)).rejects.toBeInstanceOf(FileProvisionOutcomeUnknownError);
+ expect(storage.releaseFileProvision).not.toHaveBeenCalled();
+});
+
+it('bounds cause traversal and does not override an explicit ambiguous response', () => {
+ let error: unknown = { response: { status: 403 } };
+ for (let index = 0; index < 3; index++) error = new Error('wrapped', { cause: error });
+ expect(isDefinitiveFileStorageFailure(error)).toBe(false);
+ const cyclic: { cause?: unknown } = {};
+ cyclic.cause = cyclic;
+ expect(isDefinitiveFileStorageFailure(cyclic)).toBe(false);
+ expect(
+ isDefinitiveFileStorageFailure({
+ response: { status: 500 },
+ cause: { response: { status: 403 } },
+ }),
+ ).toBe(false);
+});
+
+it.each(storageStatusErrors(403))(
+ 'keeps confirmed primary removal fenced after an SDK denial: %j',
+ async (error) => {
+ const { storage } = setup();
+ await expect(
+ withFileDeletion({
+ file,
+ userId: 'caller',
+ storage,
+ operation: async (primaryDeleted) => {
+ primaryDeleted(true);
+ throw new Error('reconciliation failed', {
+ cause: error,
+ });
+ },
+ }),
+ ).rejects.toThrow('reconciliation failed');
+ expect(storage.releaseFileDeletion).toHaveBeenCalledWith(
+ expect.objectContaining({ retainDeletion: true }),
+ );
+ },
+);
+
+it('retains any unknown secondary outcome even when another copy reports a definitive denial', async () => {
+ const { storage } = setup();
+ await expect(
+ withFileDeletion({
+ file,
+ userId: 'caller',
+ storage,
+ operation: async (_primary, onOutcomeUnknown) => {
+ onOutcomeUnknown();
+ throw Object.assign(new Error('other copy denied'), { response: { status: 403 } });
+ },
+ }),
+ ).rejects.toThrow('other copy denied');
+ expect(storage.releaseFileDeletion).toHaveBeenCalledWith(
+ expect.objectContaining({ retainDeletion: true }),
+ );
+});
+
+it.each([408, 409, 500, 503, '403', undefined].flatMap(storageStatusErrors))(
+ 'does not let a safe code or deeper denial override an explicit unknown SDK status: %j',
+ (error) => {
+ expect(
+ isDefinitiveFileStorageFailure({
+ ...error,
+ code: 'EACCES',
+ cause: { response: { status: 403 } },
+ }),
+ ).toBe(false);
+ },
+);
+
+it.each([
+ { response: { status: 500 }, $metadata: { httpStatusCode: 403 }, statusCode: 403, status: 403 },
+ { $metadata: { httpStatusCode: 500 }, statusCode: 403, status: 403 },
+ { statusCode: 500, status: 403 },
+])('prioritizes SDK HTTP contracts in adapter order: %j', (error) => {
+ expect(isDefinitiveFileStorageFailure(error)).toBe(false);
+});
+
+describe('deletion token recovery', () => {
+ it.each([true, false])(
+ 'reconciles a lost acquisition acknowledgement when applied=%s',
+ async (applied) => {
+ const { storage } = setup();
+ const acquire = storage.acquireFileDeletion.getMockImplementation()!;
+ const failure = new Error('acquisition acknowledgement lost');
+ storage.acquireFileDeletion.mockImplementationOnce(async (claim) => {
+ if (applied) await acquire(claim);
+ throw failure;
+ });
+ const stale = jest.fn();
+ await expect(
+ withFileDeletion({ file, userId: 'caller', storage, operation: stale }),
+ ).rejects.toBe(failure);
+ const original = storage.acquireFileDeletion.mock.calls[0][0];
+ expect(storage.releaseFileDeletion).not.toHaveBeenCalled();
+ const current = jest.fn(async (removed: () => void) => {
+ removed();
+ return 'current';
+ });
+ await expect(
+ withFileDeletion({ file, userId: 'caller', storage: { ...storage }, operation: current }),
+ ).resolves.toBe('current');
+ expect(stale).not.toHaveBeenCalled();
+ expect(current).toHaveBeenCalledTimes(1);
+ expect(storage.releaseFileDeletion.mock.calls[0][0]).toEqual({
+ ...original,
+ retainDeletion: true,
+ });
+ expect(storage.acquireFileDeletion.mock.calls[1][0].token).not.toBe(original.token);
+ expect(storage.releaseFileDeletion.mock.invocationCallOrder[0]).toBeLessThan(
+ storage.acquireFileDeletion.mock.invocationCallOrder[1],
+ );
+ },
+ );
+
+ it.each(['throw', 'false', 'lost-ack'] as const)(
+ 'recovers %s release after success without repeating primary I/O',
+ async (failure) => {
+ const { storage } = setup();
+ const release = storage.releaseFileDeletion.getMockImplementation()!;
+ storage.releaseFileDeletion.mockImplementationOnce(async (claim) => {
+ if (failure === 'lost-ack') await release(claim);
+ if (failure === 'false') return false;
+ throw new Error('release unavailable');
+ });
+ const value = { primary: 'removed' };
+ const operation = jest.fn(async (removed: () => void) => {
+ removed();
+ return value;
+ });
+ const options = { file, userId: 'caller', storage, operation };
+ await expect(withFileDeletion(options)).rejects.toThrow();
+ const replacement = jest.fn(async () => ({ primary: 'wrong' }));
+ await expect(withFileDeletion({ ...options, operation: replacement })).resolves.toBe(value);
+ expect(operation).toHaveBeenCalledTimes(1);
+ expect(replacement).not.toHaveBeenCalled();
+ expect(storage.acquireFileDeletion).toHaveBeenCalledTimes(1);
+ expect(storage.releaseFileDeletion.mock.calls[1][0]).toEqual(
+ storage.releaseFileDeletion.mock.calls[0][0],
+ );
+ await expect(provision(storage)).rejects.toThrow('conflicts');
+ },
+ );
+
+ it.each([false, true])(
+ 'preserves operation failure while release recovery is pending, progress=%s',
+ async (progress) => {
+ const { storage } = setup();
+ storage.releaseFileDeletion.mockResolvedValueOnce(false).mockResolvedValueOnce(false);
+ const failure = new Error('original failure');
+ const operation = jest.fn(async (removed: () => void) => {
+ if (progress) removed();
+ throw failure;
+ });
+ const options = { file, userId: 'caller', storage, operation };
+ await expect(withFileDeletion(options)).rejects.toBe(failure);
+ const original = storage.acquireFileDeletion.mock.calls[0][0];
+ const upload = jest.fn(async () => result());
+ await expect(provision(storage, upload)).rejects.toThrow('conflicts');
+ const current = jest.fn(async () => 'current');
+ await expect(withFileDeletion({ ...options, operation: current })).rejects.toThrow(
+ 'could not be released',
+ );
+ expect(current).not.toHaveBeenCalled();
+ expect(storage.acquireFileDeletion).toHaveBeenCalledTimes(1);
+ await expect(withFileDeletion({ ...options, operation: current })).resolves.toBe('current');
+ expect(storage.releaseFileDeletion.mock.calls[2][0]).toEqual({
+ ...original,
+ retainDeletion: progress,
+ });
+ expect(operation).toHaveBeenCalledTimes(1);
+ expect(current).toHaveBeenCalledTimes(1);
+ expect(upload).not.toHaveBeenCalled();
+ if (progress) await expect(provision(storage)).rejects.toThrow('conflicts');
+ else await expect(provision(storage)).resolves.toBe('uploaded');
+ },
+ );
+
+ it('does not reopen provisioning after a preflight release applied but lost its acknowledgement', async () => {
+ const { storage } = setup();
+ const release = storage.releaseFileDeletion.getMockImplementation()!;
+ storage.releaseFileDeletion.mockImplementationOnce(async (claim) => {
+ await release(claim);
+ throw new Error('release acknowledgement lost');
+ });
+ await expect(
+ withFileDeletion({
+ file,
+ userId: 'caller',
+ storage,
+ operation: async () => {
+ throw new Error('preflight');
+ },
+ }),
+ ).rejects.toThrow('preflight');
+ const upload = jest.fn(async () => result());
+ await expect(provision(storage, upload)).rejects.toThrow('conflicts');
+ expect(upload).not.toHaveBeenCalled();
+ await withFileDeletion({ file, userId: 'caller', storage, operation: async () => undefined });
+ await expect(provision(storage)).resolves.toBe('uploaded');
+ });
+
+ it.each(['acquire', 'operation', 'release'] as const)(
+ 'rejects overlap immediately during %s',
+ async (phase) => {
+ const { storage } = setup();
+ const entered = barrier();
+ const done = barrier();
+ const pause = async () => {
+ entered.release();
+ await done.wait;
+ };
+ if (phase === 'acquire') {
+ const acquire = storage.acquireFileDeletion.getMockImplementation()!;
+ storage.acquireFileDeletion.mockImplementationOnce(async (claim) => {
+ await pause();
+ return acquire(claim);
+ });
+ }
+ if (phase === 'release') {
+ const release = storage.releaseFileDeletion.getMockImplementation()!;
+ storage.releaseFileDeletion.mockImplementationOnce(async (claim) => {
+ await pause();
+ return release(claim);
+ });
+ }
+ const first = withFileDeletion({
+ file,
+ userId: 'caller',
+ storage,
+ operation: async () => {
+ if (phase === 'operation') await pause();
+ },
+ });
+ await entered.wait;
+ const replacement = jest.fn();
+ await expect(
+ withFileDeletion({ file, userId: 'caller', storage, operation: replacement }),
+ ).rejects.toThrow('conflicts');
+ expect(replacement).not.toHaveBeenCalled();
+ expect(storage.acquireFileDeletion).toHaveBeenCalledTimes(1);
+ done.release();
+ await first;
+ },
+ );
+
+ it('registers the claim before a synchronous adapter reenters deletion', async () => {
+ const { storage } = setup();
+ let reentrant: Promise<unknown> | undefined;
+ storage.acquireFileDeletion.mockImplementationOnce(() => {
+ reentrant = withFileDeletion({ file, userId: 'caller', storage, operation: jest.fn() }).catch(
+ (error: unknown) => error,
+ );
+ throw new Error('synchronous acquisition failure');
+ });
+ await expect(
+ withFileDeletion({ file, userId: 'caller', storage, operation: jest.fn() }),
+ ).rejects.toThrow('synchronous acquisition failure');
+ expect(await reentrant).toMatchObject({ message: expect.stringContaining('conflicts') });
+ expect(storage.acquireFileDeletion).toHaveBeenCalledTimes(1);
+ expect(storage.releaseFileDeletion).not.toHaveBeenCalled();
+ });
+
+ it.each([
+ { ...file, file_id: 'other-file' },
+ { ...file, user: 'other-owner' },
+ { ...file, tenantId: 'other-tenant' },
+ ])('isolates deletion recovery from %j', async (other) => {
+ const { storage } = setup();
+ storage.releaseFileDeletion.mockResolvedValueOnce(false);
+ const value = {};
+ await expect(
+ withFileDeletion({ file, userId: 'caller', storage, operation: async () => value }),
+ ).rejects.toThrow();
+ await expect(
+ withFileDeletion({ file: other, userId: 'caller', storage, operation: async () => 'other' }),
+ ).resolves.toBe('other');
+ await expect(
+ withFileDeletion({ file, userId: 'caller', storage, operation: jest.fn() }),
+ ).resolves.toBe(value);
+ });
+
+ it('never releases or steals a foreign deletion token, including after restart', async () => {
+ const { storage, deleting } = setup();
+ deleting.set(key({ ...scope, token: 'foreign' }), 'foreign');
+ const operation = jest.fn();
+ for (const recovery of [storage.recovery, createFileProvisionRecovery()]) {
+ await expect(
+ withFileDeletion({ file, userId: 'caller', storage: { ...storage, recovery }, operation }),
+ ).rejects.toThrow('conflicts');
+ }
+ expect(storage.releaseFileDeletion).not.toHaveBeenCalled();
+ expect(operation).not.toHaveBeenCalled();
+ expect([...deleting.values()]).toEqual(['foreign']);
+ });
+});
diff --git a/packages/api/src/files/operations.ts b/packages/api/src/files/operations.ts
new file mode 100644
index 0000000000..6488648b4e
--- /dev/null
+++ b/packages/api/src/files/operations.ts
@@ -0,0 +1,451 @@
+import { randomUUID } from 'node:crypto';
+import type {
+ FileStorageMethods,
+ FileStorageClaim,
+ FileStorageScope,
+ FileStorageDeleteClaim,
+ FileProvisionCommit,
+} from '@librechat/data-schemas';
+
+export type FileStorageUploadObserver = (outcome?: 'rejected') => void;
+
+export type RawProvisionStorage = Pick<
+ FileStorageMethods,
+ 'acquireFileProvision' | 'releaseFileProvision' | 'commitFileProvision'
+>;
+export type FileProvisionStorage = RawProvisionStorage & { recovery: FileProvisionRecovery };
+export type FileDeleteStorage = FileProvisionStorage &
+ Pick<FileStorageMethods, 'acquireFileDeletion' | 'releaseFileDeletion'>;
+
+type StorageIdentity = {
+ file_id: string;
+ user?: { toString(): string };
+ tenantId?: string | null;
+};
+
+export type FileProvisionResult<T> = {
+ value: T;
+ commit: Omit<FileProvisionCommit, 'claim'>;
+};
+
+type ProvisionRun<T> = {
+ file: StorageIdentity;
+ userId: string;
+ storage: RawProvisionStorage;
+ destination: string;
+ upload: (
+ claim: FileStorageClaim,
+ onUploadStart: FileStorageUploadObserver,
+ ) => Promise<FileProvisionResult<T>>;
+};
+
+type DeletionRun<T> = {
+ file: StorageIdentity & { updatedAt?: Date | string };
+ userId: string;
+ storage: FileDeleteStorage;
+ operation: (
+ primaryDeleted: (confirmed?: boolean) => void,
+ onOutcomeUnknown: () => void,
+ ) => Promise<T>;
+};
+
+type DeletionEntry = {
+ claim: FileStorageDeleteClaim;
+ phase: 'acquire' | 'operation' | 'release';
+ active: boolean;
+ retainDeletion: boolean;
+ resumed: boolean;
+ primaryConfirmed: boolean;
+ uncertain: boolean;
+ hasResult: boolean;
+ value?: unknown;
+};
+
+export interface FileProvisionRecovery {
+ delete<T>(options: DeletionRun<T>): Promise<T>;
+ run<T>(options: ProvisionRun<T>): Promise<T>;
+ recover(scope: FileStorageScope, storage: RawProvisionStorage): Promise<void>;
+}
+
+/** The remote copy exists but its durable pointer is not confirmed. Keep the claim for recovery. */
+export class FileProvisionPersistenceError extends Error {
+ constructor(cause?: unknown) {
+ super('File provisioning persistence was not confirmed', { cause });
+ this.name = 'FileProvisionPersistenceError';
+ }
+}
+
+/** Dispatch may have created a remote copy. Only operator-verified absence can release its claim. */
+export class FileProvisionOutcomeUnknownError extends FileProvisionPersistenceError {
+ constructor(cause?: unknown) {
+ super(cause);
+ this.name = 'FileProvisionOutcomeUnknownError';
+ this.message = 'File provisioning outcome is unknown; manual recovery is required';
+ }
+}
+
+/** Only explicit denials establish that storage did not accept the operation. */
+export function isDefinitiveFileStorageFailure(error: unknown): boolean {
+ let current = error;
+ // RAG wraps the HTTP failure; inspect at most two nested causes.
+ for (let depth = 0; depth < 3; depth++) {
+ if (typeof current !== 'object' || current === null) return false;
+ let status: unknown;
+ let hasStatus = true;
+ if (
+ 'response' in current &&
+ typeof current.response === 'object' &&
+ current.response !== null &&
+ 'status' in current.response
+ ) {
+ status = current.response.status;
+ } else if (
+ '$metadata' in current &&
+ typeof current.$metadata === 'object' &&
+ current.$metadata !== null &&
+ 'httpStatusCode' in current.$metadata
+ ) {
+ status = current.$metadata.httpStatusCode;
+ } else if ('statusCode' in current) {
+ status = current.statusCode;
+ } else if ('status' in current) {
+ status = current.status;
+ } else {
+ hasStatus = false;
+ }
+ if (hasStatus) {
+ switch (status) {
+ case 400:
+ case 401:
+ case 403:
+ case 404:
+ case 413:
+ case 415:
+ case 422:
+ case 429:
+ return true;
+ default:
+ return false;
+ }
+ }
+ if ('code' in current) {
+ switch (current.code) {
+ case 'EACCES':
+ case 'EPERM':
+ case 'CODE_API_RATE_LIMITED':
+ return true;
+ }
+ }
+ if (!('cause' in current)) return false;
+ current = current.cause;
+ }
+ return false;
+}
+
+type ProvisionEntry = {
+ claim: FileStorageClaim;
+ phase: 'upload' | 'commit' | 'release' | 'ambiguous';
+ hasResult: boolean;
+ value?: unknown;
+ commit?: FileProvisionCommit;
+ attempt?: Promise<unknown>;
+ recovering?: boolean;
+};
+
+function scopeKey(scope: FileStorageScope): string {
+ return JSON.stringify([scope.tenantId ?? null, scope.userId, scope.file_id]);
+}
+
+function fileScope(file: StorageIdentity, userId: string): FileStorageScope {
+ return {
+ file_id: file.file_id,
+ userId: file.user?.toString() ?? userId,
+ tenantId: file.tenantId,
+ };
+}
+
+/** App-lifetime recovery only. Unknown claims, including those lost on restart, need manual recovery. */
+export function createFileProvisionRecovery(): FileProvisionRecovery {
+ const files = new Map<string, Map<string, ProvisionEntry>>();
+ const deletions = new Map<string, DeletionEntry>();
+
+ function remove(key: string, destination: string, entry: ProvisionEntry): void {
+ const entries = files.get(key);
+ if (entries?.get(destination) !== entry) return;
+ entries.delete(destination);
+ if (entries.size === 0) files.delete(key);
+ }
+
+ async function release(entry: ProvisionEntry, storage: RawProvisionStorage): Promise<void> {
+ try {
+ if (!(await storage.releaseFileProvision(entry.claim))) {
+ throw new FileProvisionPersistenceError();
+ }
+ } catch (error) {
+ if (error instanceof FileProvisionPersistenceError) throw error;
+ throw new FileProvisionPersistenceError(error);
+ }
+ }
+
+ async function settle(
+ key: string,
+ destination: string,
+ entry: ProvisionEntry,
+ storage: RawProvisionStorage,
+ ): Promise<unknown> {
+ if (entry.phase === 'ambiguous') throw new FileProvisionOutcomeUnknownError();
+ if (entry.phase === 'commit' && entry.commit) {
+ for (let attempt = 0; attempt < 2; attempt++) {
+ try {
+ // Storage receives a copy so a failed adapter cannot alter the recovery payload.
+ await confirmFileProvision(storage, structuredClone(entry.commit));
+ entry.phase = 'release';
+ break;
+ } catch (error) {
+ if (attempt === 1) throw error;
+ }
+ }
+ }
+ await release(entry, storage);
+ remove(key, destination, entry);
+ return entry.value;
+ }
+
+ function retry(
+ key: string,
+ destination: string,
+ entry: ProvisionEntry,
+ storage: RawProvisionStorage,
+ ): Promise<unknown> {
+ if (entry.attempt) return entry.attempt;
+ entry.recovering = true;
+ entry.attempt = Promise.resolve()
+ .then(() => settle(key, destination, entry, storage))
+ .finally(() => {
+ entry.attempt = undefined;
+ entry.recovering = false;
+ });
+ return entry.attempt;
+ }
+
+ async function run<T>(options: ProvisionRun<T>): Promise<T> {
+ const { file, userId, storage, destination, upload } = options;
+ const scope = fileScope(file, userId);
+ const key = scopeKey(scope);
+ if (deletions.has(key)) throw new Error('File provisioning conflicts with deletion');
+ let entries = files.get(key);
+ const existing = entries?.get(destination);
+ if (existing) {
+ const reusable = existing.phase !== 'release' || existing.hasResult;
+ const value = await (existing.attempt ?? retry(key, destination, existing, storage));
+ // Only values from this exact scope and destination cross the internal generic map.
+ if (reusable) return value as T;
+ return run(options);
+ }
+ if (!entries) {
+ entries = new Map();
+ files.set(key, entries);
+ }
+ const entry: ProvisionEntry = {
+ claim: Object.freeze({ ...scope, token: randomUUID() }),
+ phase: 'upload',
+ hasResult: false,
+ };
+ entries.set(destination, entry);
+ const execute = async (): Promise<T> => {
+ let acquired: boolean;
+ try {
+ acquired = await storage.acquireFileProvision(entry.claim);
+ } catch (error) {
+ // The claim may be durable even when its acknowledgement was lost.
+ entry.phase = 'release';
+ throw new FileProvisionPersistenceError(error);
+ }
+ if (!acquired) {
+ remove(key, destination, entry);
+ throw new Error('File provisioning conflicts with deletion or a missing record');
+ }
+ let result: FileProvisionResult<T>;
+ let uploadStarted = false;
+ try {
+ result = await upload(entry.claim, (outcome) => {
+ uploadStarted = outcome !== 'rejected';
+ });
+ } catch (error) {
+ if (uploadStarted && !isDefinitiveFileStorageFailure(error)) {
+ entry.phase = 'ambiguous';
+ throw new FileProvisionOutcomeUnknownError(error);
+ }
+ entry.phase = 'release';
+ try {
+ await release(entry, storage);
+ remove(key, destination, entry);
+ } catch {
+ // Preserve the upload error while retaining release-only recovery.
+ }
+ throw error;
+ }
+ entry.value = result.value;
+ entry.hasResult = true;
+ entry.commit = { ...structuredClone(result.commit), claim: entry.claim };
+ entry.phase = 'commit';
+ await settle(key, destination, entry, storage);
+ return result.value;
+ };
+ entry.attempt = Promise.resolve()
+ .then(execute)
+ .finally(() => {
+ entry.attempt = undefined;
+ });
+ return entry.attempt as Promise<T>;
+ }
+
+ async function releaseDeletion(entry: DeletionEntry, storage: FileDeleteStorage): Promise<void> {
+ if (
+ !(await storage.releaseFileDeletion({
+ ...entry.claim,
+ retainDeletion: entry.retainDeletion,
+ }))
+ ) {
+ throw new Error('File deletion claim could not be released');
+ }
+ }
+
+ async function runDeletion<T>(options: DeletionRun<T>): Promise<T> {
+ const { file, userId, storage, operation } = options;
+ const scope = fileScope(file, userId);
+ const key = scopeKey(scope);
+ const existing = deletions.get(key);
+ if (existing?.active) throw new Error('File deletion conflicts with active storage work');
+ if (existing) {
+ existing.active = true;
+ try {
+ // An unacknowledged acquisition may have resumed an older deletion intent.
+ if (existing.phase === 'acquire') existing.retainDeletion = true;
+ await releaseDeletion(existing, storage);
+ if (existing.hasResult) {
+ deletions.delete(key);
+ return existing.value as T;
+ }
+ } finally {
+ existing.active = false;
+ }
+ }
+ const entry: DeletionEntry = {
+ claim: Object.freeze({ ...scope, updatedAt: file.updatedAt, token: randomUUID() }),
+ phase: 'acquire',
+ active: true,
+ retainDeletion: true,
+ resumed: false,
+ primaryConfirmed: false,
+ uncertain: false,
+ hasResult: false,
+ };
+ // Register before any adapter runs, including synchronous/reentrant adapters.
+ deletions.set(key, entry);
+ try {
+ try {
+ await storage.recovery.recover(scope, storage);
+ } catch (error) {
+ deletions.delete(key);
+ throw error;
+ }
+ const acquired = await storage.acquireFileDeletion(entry.claim);
+ if (!acquired) {
+ deletions.delete(key);
+ throw new Error('File deletion conflicts with active storage work');
+ }
+ entry.resumed = acquired.resumed;
+ entry.retainDeletion = acquired.resumed;
+ entry.phase = 'operation';
+ let value: T;
+ try {
+ value = await operation(
+ (confirmed = true) => {
+ entry.retainDeletion = true;
+ entry.primaryConfirmed = entry.primaryConfirmed || confirmed;
+ },
+ () => {
+ entry.uncertain = true;
+ entry.retainDeletion = true;
+ },
+ );
+ } catch (error) {
+ if (
+ entry.retainDeletion &&
+ !entry.resumed &&
+ !entry.primaryConfirmed &&
+ !entry.uncertain &&
+ isDefinitiveFileStorageFailure(error)
+ ) {
+ entry.retainDeletion = false;
+ }
+ entry.phase = 'release';
+ try {
+ await releaseDeletion(entry, storage);
+ deletions.delete(key);
+ } catch {
+ // Preserve the operation failure and retain the original release token.
+ }
+ throw error;
+ }
+ entry.value = value;
+ entry.hasResult = true;
+ entry.phase = 'release';
+ await releaseDeletion(entry, storage);
+ deletions.delete(key);
+ return value;
+ } finally {
+ entry.active = false;
+ }
+ }
+
+ return {
+ run,
+ delete: runDeletion,
+ async recover(scope, storage) {
+ const key = scopeKey(scope);
+ const entries = files.get(key);
+ if (!entries) return;
+ const attempts: Promise<unknown>[] = [];
+ for (const [destination, entry] of entries) {
+ if (entry.phase === 'upload' || (entry.attempt && !entry.recovering)) continue;
+ attempts.push(retry(key, destination, entry, storage));
+ }
+ await Promise.all(attempts);
+ },
+ };
+}
+
+/** A durable claim spans upload, pointer persistence and release. */
+export function withFileProvisioning<T>({
+ operation,
+ storage,
+ ...options
+}: Omit<ProvisionRun<T>, 'upload' | 'storage'> & {
+ storage: FileProvisionStorage;
+ operation: (
+ claim: FileStorageClaim,
+ onUploadStart: FileStorageUploadObserver,
+ ) => Promise<FileProvisionResult<T>>;
+}): Promise<T> {
+ return storage.recovery.run({ ...options, storage, upload: operation });
+}
+
+/** Successful primary removal leaves deletion intent until conditional metadata finalization. */
+export function withFileDeletion<T>(options: DeletionRun<T>): Promise<T> {
+ return options.storage.recovery.delete(options);
+}
+
+/** A missing or replaced claim cannot publish a newly uploaded copy. */
+export async function confirmFileProvision(
+ storage: Pick<RawProvisionStorage, 'commitFileProvision'>,
+ data: FileProvisionCommit,
+): Promise<void> {
+ try {
+ if (!(await storage.commitFileProvision(data))) throw new FileProvisionPersistenceError();
+ } catch (error) {
+ if (error instanceof FileProvisionPersistenceError) throw error;
+ throw new FileProvisionPersistenceError(error);
+ }
+}
diff --git a/packages/api/src/files/provision/callback.spec.ts b/packages/api/src/files/provision/callback.spec.ts
index 28846bdb92..a69664039f 100644
--- a/packages/api/src/files/provision/callback.spec.ts
+++ b/packages/api/src/files/provision/callback.spec.ts
@@ -1,10 +1,12 @@
import { Constants } from '@librechat/agents';
-import { EToolResources, FileContext } from 'librechat-data-provider';
-import type { TFile } from 'librechat-data-provider';
+import { FileSources, FileContext, EToolResources } from 'librechat-data-provider';
+import type { FileProvisionCommit, FileStorageClaim } from '@librechat/data-schemas';
+import type { CodeEnvRef, TFile } from 'librechat-data-provider';
+import type { ProvisionCallbackDeps, ProvisionToolContext } from './callback';
import type { ProvisionState } from '~/agents/resources';
-import type { ProvisionToolContext } from './callback';
import type { CodeFileAgent } from '../code/queued';
import type { ServerRequest } from '~/types';
+import { createFileProvisionRecovery, withFileProvisioning } from '../operations';
import { mergeCodeFilesIntoContext } from '~/agents/codeFilesSession';
import { prepareQueuedCodeFileContext } from '../code/queued';
import { createSubagentCodeRouting } from '~/code/targets';
@@ -16,6 +18,15 @@ jest.mock('@librechat/data-schemas', () => ({
const req = { user: { id: 'user-1' } } as ServerRequest;
+function wrapUpload<TArgs extends unknown[], T>(
+ mock: jest.MockInstance<Promise<T>, TArgs>,
+ run: (execute: () => Promise<T>) => Promise<T>,
+): void {
+ const original = mock.getMockImplementation();
+ if (!original) throw new Error('Missing upload implementation');
+ mock.mockImplementation((...args: TArgs) => run(() => original(...args)));
+}
+
function makeFile(overrides: Partial<TFile> = {}): TFile {
return {
file_id: 'file-1',
@@ -58,9 +69,12 @@ function buildHarness({
typeof createProvisionFilesCallback
>[0]['resolveExecutionContext'];
}) {
- const provisionToCodeEnv =
+ const provisionToCodeEnv: jest.MockedFunction<ProvisionCallbackDeps['provisionToCodeEnv']> =
codeImpl ??
- jest.fn(async ({ file, sandboxFilename }: { file: TFile; sandboxFilename: string }) => ({
+ jest.fn<
+ ReturnType<ProvisionCallbackDeps['provisionToCodeEnv']>,
+ Parameters<ProvisionCallbackDeps['provisionToCodeEnv']>
+ >(async ({ file, sandboxFilename }) => ({
referenceSet: {
codeEnvRefs: {
default: {
@@ -78,17 +92,45 @@ function buildHarness({
},
/* Mirrors the service: the sandbox stores a converted image under a corrected
* extension, so the name the model is told is not always the record's. */
- sandboxFilename,
+ sandboxFilename: sandboxFilename ?? file.filename,
}));
- const provisionToVectorDB =
+ const provisionToVectorDB: jest.MockedFunction<ProvisionCallbackDeps['provisionToVectorDB']> =
vectorImpl ??
- jest.fn(async ({ file }: { file: TFile }) => ({
+ jest.fn<
+ ReturnType<ProvisionCallbackDeps['provisionToVectorDB']>,
+ Parameters<ProvisionCallbackDeps['provisionToVectorDB']>
+ >(async ({ file }) => ({
embedded: true,
fileUpdate: { file_id: file.file_id, embedded: true },
}));
- const updateFile = jest.fn(async () => ({}));
- const updateCodeEnvRef = jest.fn(async () => ({}));
- const addEmbeddedEntity = jest.fn(async () => ({}));
+ const updateFile = jest.fn(
+ async (_update: Parameters<ProvisionCallbackDeps['updateFile']>[0]): Promise<unknown> => ({}),
+ );
+ const updateCodeEnvRef = jest.fn(
+ async (
+ _update: Parameters<ProvisionCallbackDeps['updateCodeEnvRef']>[0],
+ ): Promise<unknown> => ({}),
+ );
+ const addEmbeddedEntity = jest.fn(
+ async (
+ _update: Parameters<ProvisionCallbackDeps['addEmbeddedEntity']>[0],
+ ): Promise<unknown> => ({}),
+ );
+ const fileStorage = {
+ recovery: createFileProvisionRecovery(),
+ acquireFileProvision: jest.fn(async (_claim: FileStorageClaim) => true),
+ releaseFileProvision: jest.fn(async (_claim: FileStorageClaim) => true),
+ commitFileProvision: jest.fn(
+ async ({ claim, code, entityId, embedded }: FileProvisionCommit) => {
+ let result: unknown;
+ if (code) result = await updateCodeEnvRef({ file_id: claim.file_id, ...code });
+ else if (entityId != null)
+ result = await addEmbeddedEntity({ file_id: claim.file_id, entityId });
+ else result = await updateFile({ file_id: claim.file_id, embedded });
+ return result !== null && result !== false;
+ },
+ ),
+ };
const agentToolContexts = new Map<string, ProvisionToolContext>(contexts);
return {
@@ -98,8 +140,10 @@ function buildHarness({
updateCodeEnvRef,
addEmbeddedEntity,
agentToolContexts,
+ fileStorage,
provisionFiles: createProvisionFilesCallback({
req,
+ fileStorage,
agentToolContexts,
provisionToCodeEnv: provisionToCodeEnv as never,
provisionToVectorDB: provisionToVectorDB as never,
@@ -112,6 +156,328 @@ function buildHarness({
}
describe('createProvisionFilesCallback', () => {
+ it.each(['primer', 'callback'] as const)(
+ 'recovers the canonical ref across producers when the %s uploads first',
+ async (first) => {
+ const oldRef: CodeEnvRef = {
+ kind: 'user',
+ id: 'user-1',
+ storage_session_id: 'other-session',
+ file_id: 'other-file',
+ executionProfile: 'stateful',
+ };
+ const userId = 'user-1';
+ const file = makeFile({ metadata: { codeEnvRef: oldRef, destinationChosen: false } });
+ const provisionState = state([file], []);
+ const h = buildHarness({ contexts: [['agent-a', { provisionState }]] });
+ const ref: CodeEnvRef = {
+ kind: 'user',
+ id: 'user-1',
+ storage_session_id: 'stored-session',
+ file_id: 'stored-file',
+ sandboxFilename: 'stored.csv',
+ };
+ h.provisionToCodeEnv.mockResolvedValue({
+ referenceSet: { codeEnvRefs: { default: ref } },
+ refUpdate: { file_id: file.file_id, routeKey: 'default', ref },
+ sandboxFilename: ref.sandboxFilename!,
+ });
+ const upload = jest.fn(async () => ({
+ value: ref,
+ commit: { code: { routeKey: 'default', ref } },
+ }));
+ const primer = () =>
+ withFileProvisioning({
+ file,
+ userId,
+ storage: h.fileStorage,
+ destination: JSON.stringify(['code', 'default', 'user', userId, null, file.filename]),
+ operation: upload,
+ });
+ const callback = () => h.provisionFiles([Constants.EXECUTE_CODE], 'agent-a');
+ h.fileStorage.commitFileProvision.mockResolvedValueOnce(false).mockResolvedValueOnce(false);
+ await expect(first === 'primer' ? primer() : callback()).rejects.toThrow();
+ const claim = h.fileStorage.acquireFileProvision.mock.calls[0][0];
+ if (first === 'primer') {
+ await expect(callback()).resolves.toEqual([
+ {
+ id: ref.file_id,
+ resource_id: ref.id,
+ storage_session_id: ref.storage_session_id,
+ name: ref.sandboxFilename,
+ kind: ref.kind,
+ },
+ ]);
+ expect(h.provisionToCodeEnv).not.toHaveBeenCalled();
+ expect(file.metadata).toMatchObject({
+ destinationChosen: false,
+ codeEnvRefs: { default: ref, stateful: oldRef },
+ });
+ expect(provisionState.codeEnvFiles).toEqual([]);
+ } else {
+ await expect(primer()).resolves.toEqual(ref);
+ expect(upload).not.toHaveBeenCalled();
+ }
+ expect(h.fileStorage.acquireFileProvision).toHaveBeenCalledTimes(1);
+ expect(h.fileStorage.commitFileProvision).toHaveBeenLastCalledWith(
+ expect.objectContaining({ claim, code: expect.objectContaining({ ref }) }),
+ );
+ expect(h.fileStorage.releaseFileProvision).toHaveBeenCalledWith(claim);
+ },
+ );
+
+ describe.each(['code', 'vector'] as const)('%s provisioning claims', (destination) => {
+ const toolNames = destination === 'code' ? [Constants.EXECUTE_CODE] : ['file_search'];
+ function claimHarness() {
+ const file = makeFile({ tenantId: 'tenant-1' });
+ const provisionState = state(
+ destination === 'code' ? [file] : [],
+ destination === 'vector' ? [file] : [],
+ );
+ const harness = buildHarness({ contexts: [['agent-a', { provisionState }]] });
+ return {
+ ...harness,
+ file,
+ provisionState,
+ upload: destination === 'code' ? harness.provisionToCodeEnv : harness.provisionToVectorDB,
+ persist: destination === 'code' ? harness.updateCodeEnvRef : harness.addEmbeddedEntity,
+ };
+ }
+
+ it('rejects an unauthenticated provisioning principal before acquiring storage', async () => {
+ const harness = claimHarness();
+ const provisionFiles = createProvisionFilesCallback({
+ req: {} as ServerRequest,
+ fileStorage: harness.fileStorage,
+ agentToolContexts: harness.agentToolContexts,
+ provisionToCodeEnv: harness.provisionToCodeEnv as never,
+ provisionToVectorDB: harness.provisionToVectorDB as never,
+ updateFile: harness.updateFile,
+ updateCodeEnvRef: harness.updateCodeEnvRef,
+ addEmbeddedEntity: harness.addEmbeddedEntity,
+ });
+ await expect(provisionFiles(toolNames, 'agent-a')).rejects.toThrow(
+ 'File provisioning requires an authenticated user',
+ );
+ expect(harness.fileStorage.acquireFileProvision).not.toHaveBeenCalled();
+ expect(harness.upload).not.toHaveBeenCalled();
+ });
+
+ it('does not read credentials or upload when the durable claim is denied', async () => {
+ const harness = claimHarness();
+ const readKey = jest.fn();
+ harness.upload.mockImplementation(readKey);
+ harness.fileStorage.acquireFileProvision.mockResolvedValueOnce(false);
+
+ await expect(harness.provisionFiles(toolNames, 'agent-a')).rejects.toThrow(
+ 'Failed to provision',
+ );
+
+ expect(harness.fileStorage.acquireFileProvision).toHaveBeenCalledTimes(1);
+ expect(readKey).not.toHaveBeenCalled();
+ expect(harness.provisionToCodeEnv).not.toHaveBeenCalled();
+ expect(harness.provisionToVectorDB).not.toHaveBeenCalled();
+ expect(harness.persist).not.toHaveBeenCalled();
+ expect(harness.fileStorage.releaseFileProvision).not.toHaveBeenCalled();
+ expect([
+ ...harness.provisionState.codeEnvFiles,
+ ...harness.provisionState.vectorDBFiles,
+ ]).toEqual([harness.file]);
+ });
+
+ it('holds the claim through paused upload and pointer persistence, then releases it', async () => {
+ const harness = claimHarness();
+ let finishUpload!: () => void;
+ let finishWrite!: () => void;
+ const uploadGate = new Promise<void>((resolve) => {
+ finishUpload = resolve;
+ });
+ const writeGate = new Promise<void>((resolve) => {
+ finishWrite = resolve;
+ });
+ const paused = async <T>(execute: () => Promise<T>): Promise<T> => {
+ expect(harness.fileStorage.acquireFileProvision).toHaveBeenCalledTimes(1);
+ await uploadGate;
+ return execute();
+ };
+ if (destination === 'code') wrapUpload(harness.provisionToCodeEnv, paused);
+ else wrapUpload(harness.provisionToVectorDB, paused);
+ harness.persist.mockImplementation(async () => {
+ await writeGate;
+ return {};
+ });
+ const pending = harness.provisionFiles(toolNames, 'agent-a');
+ await new Promise((resolve) => setImmediate(resolve));
+ expect(harness.upload).toHaveBeenCalledTimes(1);
+ expect(harness.persist).not.toHaveBeenCalled();
+ expect(harness.fileStorage.releaseFileProvision).not.toHaveBeenCalled();
+ finishUpload();
+ await new Promise((resolve) => setImmediate(resolve));
+ expect(harness.persist).toHaveBeenCalledTimes(1);
+ expect(harness.fileStorage.releaseFileProvision).not.toHaveBeenCalled();
+ finishWrite();
+ await pending;
+ expect(harness.fileStorage.releaseFileProvision).toHaveBeenCalledTimes(1);
+ expect(harness.fileStorage.commitFileProvision).toHaveBeenCalledWith(
+ expect.objectContaining({
+ claim: harness.fileStorage.acquireFileProvision.mock.calls[0][0],
+ }),
+ );
+ expect(harness.fileStorage.releaseFileProvision.mock.calls[0]).toEqual(
+ harness.fileStorage.acquireFileProvision.mock.calls[0],
+ );
+ expect(harness.fileStorage.acquireFileProvision).toHaveBeenCalledWith(
+ expect.objectContaining({
+ file_id: harness.file.file_id,
+ tenantId: 'tenant-1',
+ token: expect.any(String),
+ }),
+ );
+ });
+
+ it('records a successful remote result before honoring cancellation', async () => {
+ const harness = claimHarness();
+ const controller = new AbortController();
+ const cancelled = async <T>(execute: () => Promise<T>): Promise<T> => {
+ const result = await execute();
+ controller.abort();
+ return result;
+ };
+ if (destination === 'code') wrapUpload(harness.provisionToCodeEnv, cancelled);
+ else wrapUpload(harness.provisionToVectorDB, cancelled);
+ await expect(
+ harness.provisionFiles(toolNames, 'agent-a', controller.signal),
+ ).rejects.toMatchObject({ name: 'AbortError' });
+ expect(harness.fileStorage.commitFileProvision).toHaveBeenCalledTimes(1);
+ expect(harness.fileStorage.releaseFileProvision).toHaveBeenCalledTimes(1);
+ });
+
+ it('keeps the file queued when the conditional commit returns false', async () => {
+ const harness = claimHarness();
+ harness.fileStorage.commitFileProvision.mockResolvedValue(false);
+ await expect(harness.provisionFiles(toolNames, 'agent-a')).rejects.toThrow(
+ 'Failed to provision',
+ );
+ expect(harness.fileStorage.commitFileProvision).toHaveBeenCalledTimes(2);
+ expect(harness.fileStorage.releaseFileProvision).not.toHaveBeenCalled();
+ expect([
+ ...harness.provisionState.codeEnvFiles,
+ ...harness.provisionState.vectorDBFiles,
+ ]).toEqual([harness.file]);
+ });
+
+ it('recovers persistence in a fresh callback without another upload or token', async () => {
+ const harness = claimHarness();
+ harness.fileStorage.commitFileProvision
+ .mockResolvedValueOnce(false)
+ .mockResolvedValueOnce(false);
+ await expect(harness.provisionFiles(toolNames, 'agent-a')).rejects.toThrow(
+ 'Failed to provision',
+ );
+ const originalClaim = harness.fileStorage.acquireFileProvision.mock.calls[0][0];
+ const next = createProvisionFilesCallback({
+ req,
+ fileStorage: harness.fileStorage,
+ agentToolContexts: harness.agentToolContexts,
+ provisionToCodeEnv: harness.provisionToCodeEnv as never,
+ provisionToVectorDB: harness.provisionToVectorDB as never,
+ updateFile: harness.updateFile,
+ updateCodeEnvRef: harness.updateCodeEnvRef,
+ addEmbeddedEntity: harness.addEmbeddedEntity,
+ });
+ await next(toolNames, 'agent-a');
+ expect(harness.upload).toHaveBeenCalledTimes(1);
+ expect(harness.fileStorage.acquireFileProvision).toHaveBeenCalledTimes(1);
+ expect(harness.fileStorage.commitFileProvision).toHaveBeenLastCalledWith(
+ expect.objectContaining({ claim: originalClaim }),
+ );
+ expect(harness.fileStorage.releaseFileProvision).toHaveBeenCalledWith(originalClaim);
+ expect(harness.provisionState.codeEnvFiles).toEqual([]);
+ expect(harness.provisionState.vectorDBFiles).toEqual([]);
+ });
+
+ it('retains the claim when a dispatched upload loses its response', async () => {
+ const harness = claimHarness();
+ harness.upload.mockImplementationOnce(
+ async ({ onUploadStart }: { onUploadStart?: (outcome?: 'rejected') => void }) => {
+ onUploadStart?.();
+ throw new Error('transport response lost');
+ },
+ );
+ await expect(harness.provisionFiles(toolNames, 'agent-a')).rejects.toThrow(
+ 'Failed to provision',
+ );
+ expect(harness.fileStorage.releaseFileProvision).not.toHaveBeenCalled();
+ await expect(harness.provisionFiles(toolNames, 'agent-a')).rejects.toThrow(
+ 'Failed to provision',
+ );
+ expect(harness.upload).toHaveBeenCalledTimes(1);
+ expect(harness.fileStorage.acquireFileProvision).toHaveBeenCalledTimes(1);
+ expect(harness.fileStorage.commitFileProvision).not.toHaveBeenCalled();
+ });
+
+ it('releases a dispatched claim after a definitive access rejection', async () => {
+ const harness = claimHarness();
+ harness.upload.mockImplementationOnce(
+ async ({ onUploadStart }: { onUploadStart?: (outcome?: 'rejected') => void }) => {
+ onUploadStart?.();
+ throw Object.assign(new Error('denied'), { response: { status: 403 } });
+ },
+ );
+ await expect(harness.provisionFiles(toolNames, 'agent-a')).rejects.toThrow(
+ 'Failed to provision',
+ );
+ expect(harness.fileStorage.releaseFileProvision).toHaveBeenCalledTimes(1);
+ await harness.provisionFiles(toolNames, 'agent-a');
+ expect(harness.upload).toHaveBeenCalledTimes(2);
+ expect(harness.fileStorage.releaseFileProvision).toHaveBeenCalledTimes(2);
+ });
+
+ it('releases the claim on an upload failure and retains the queue', async () => {
+ const harness = claimHarness();
+ harness.upload.mockRejectedValueOnce(new Error('upload unavailable'));
+ await expect(harness.provisionFiles(toolNames, 'agent-a')).rejects.toThrow(
+ 'Failed to provision',
+ );
+ expect(harness.fileStorage.releaseFileProvision).toHaveBeenCalledTimes(1);
+ expect(harness.persist).not.toHaveBeenCalled();
+ expect([
+ ...harness.provisionState.codeEnvFiles,
+ ...harness.provisionState.vectorDBFiles,
+ ]).toEqual([harness.file]);
+ await harness.provisionFiles(toolNames, 'agent-a');
+ expect(harness.fileStorage.acquireFileProvision).toHaveBeenCalledTimes(2);
+ expect(harness.fileStorage.releaseFileProvision).toHaveBeenCalledTimes(2);
+ });
+
+ it.each(['reject', 'null'] as const)(
+ 'retries a %s pointer write and keeps the file queued on failure',
+ async (failure) => {
+ const harness = claimHarness();
+ if (failure === 'null') harness.persist.mockResolvedValue(null);
+ else harness.persist.mockRejectedValue(new Error('database unavailable'));
+ await expect(harness.provisionFiles(toolNames, 'agent-a')).rejects.toThrow(
+ 'Failed to provision',
+ );
+ expect(harness.persist).toHaveBeenCalledTimes(2);
+ expect(harness.fileStorage.releaseFileProvision).not.toHaveBeenCalled();
+ expect([
+ ...harness.provisionState.codeEnvFiles,
+ ...harness.provisionState.vectorDBFiles,
+ ]).toEqual([harness.file]);
+ expect(harness.file.embedded).toBe(false);
+ harness.persist.mockResolvedValue(undefined);
+ await harness.provisionFiles(toolNames, 'agent-a');
+ expect(harness.upload).toHaveBeenCalledTimes(1);
+ expect(harness.fileStorage.releaseFileProvision).toHaveBeenCalledTimes(1);
+ expect([
+ ...harness.provisionState.codeEnvFiles,
+ ...harness.provisionState.vectorDBFiles,
+ ]).toEqual([]);
+ },
+ );
+ });
+
it.each(['a', 'b'])(
'keeps distinct queued files reachable at their advertised paths when %s runs first',
async (firstAgentId) => {
@@ -421,8 +787,11 @@ describe('createProvisionFilesCallback', () => {
const calls = provisionToCodeEnv.mock.calls.map(([args]) => args);
expect(calls.filter(({ file }) => file.file_id === 'shared')).toHaveLength(1);
- expect(calls.find(({ file }) => file.file_id === 'shared').sandboxFilename).toBe('data.csv');
- expect(calls.find(({ file }) => file.file_id === 'own').sandboxFilename).not.toBe('data.csv');
+ const sharedUpload = calls.find(({ file }) => file.file_id === 'shared');
+ const ownUpload = calls.find(({ file }) => file.file_id === 'own');
+ if (!sharedUpload || !ownUpload) throw new Error('Expected both upload destinations');
+ expect(sharedUpload.sandboxFilename).toBe('data.csv');
+ expect(ownUpload.sandboxFilename).not.toBe('data.csv');
});
it('recovers a cleared reference under its saved route name', async () => {
@@ -1042,6 +1411,47 @@ describe('createProvisionFilesCallback', () => {
expect(codeImpl.mock.calls.filter(([args]) => args.file.file_id === 'first')).toHaveLength(1);
});
+ it('repairs a retained Project file before search and preserves a failed attempt for retry', async () => {
+ const file = makeFile({
+ embedded: false,
+ source: FileSources.s3,
+ metadata: { searchDestination: true },
+ });
+ const vectorImpl = jest
+ .fn()
+ .mockRejectedValueOnce(new Error('temporary vector outage'))
+ .mockImplementationOnce(async () => ({
+ embedded: true,
+ fileUpdate: { file_id: file.file_id, embedded: true },
+ }));
+ const { provisionFiles, agentToolContexts, addEmbeddedEntity } = buildHarness({
+ contexts: [
+ [
+ 'agent-a',
+ {
+ provisionState: state([], [file]),
+ tool_resources: {
+ [EToolResources.file_search]: { files: [file] },
+ },
+ },
+ ],
+ ],
+ vectorImpl,
+ });
+ await expect(provisionFiles(['file_search'], 'agent-a')).rejects.toThrow(
+ /aborting tool execution/,
+ );
+ expect(file.embedded).toBe(false);
+ expect(agentToolContexts.get('agent-a')?.provisionState?.vectorDBFiles).toEqual([file]);
+ await expect(provisionFiles(['file_search'], 'agent-a')).resolves.toEqual([]);
+ expect(vectorImpl).toHaveBeenLastCalledWith(
+ expect.objectContaining({ file, entity_id: undefined }),
+ );
+ expect(addEmbeddedEntity).toHaveBeenCalledWith({ file_id: file.file_id, entityId: 'user-1' });
+ expect(file.embedded).toBe(true);
+ expect(agentToolContexts.get('agent-a')?.provisionState?.vectorDBFiles).toEqual([]);
+ });
+
it('treats a declined embedding as a failure rather than a result', async () => {
/* The service resolves with embedded false when the vector store declines the file.
* That means the vectors are absent just as a throw does, so the file has to stay
diff --git a/packages/api/src/files/provision/callback.ts b/packages/api/src/files/provision/callback.ts
index 7aee554605..6c1984bac7 100644
--- a/packages/api/src/files/provision/callback.ts
+++ b/packages/api/src/files/provision/callback.ts
@@ -1,12 +1,14 @@
import { Constants } from '@librechat/agents';
import { logger } from '@librechat/data-schemas';
-import { EToolResources } from 'librechat-data-provider';
+import { EToolResources, mergeCodeEnvRef } from 'librechat-data-provider';
import type { CodeEnvFile, SubagentExecutionContext } from '@librechat/agents';
import type { TFile } from 'librechat-data-provider';
import type { CodeEnvRefUpdate, ProvisionService } from './service';
import type { ProvisionToolContext } from '../code/queued';
+import type { FileProvisionStorage } from '../operations';
import type { ServerRequest } from '~/types';
import { createCodeApiRateLimitBudget, isCodeApiRateLimitError } from '~/utils';
+import { withFileProvisioning } from '../operations';
import { planCodeFileUploads } from '../code/queued';
import { isCodeFileToolName } from '~/agents/tools';
@@ -17,29 +19,11 @@ interface FileUpdate {
embedded?: boolean;
}
-/** One attempt plus one retry: a transient write failure would otherwise leave the record
- * unprovisioned while the queue is cleared. */
-async function persistWithRetry(
- write: () => Promise<unknown>,
- onFailure: (error: unknown) => void,
-): Promise<boolean> {
- for (let attempt = 0; attempt < 2; attempt++) {
- try {
- await write();
- return true;
- } catch (error) {
- if (attempt === 1) {
- onFailure(error);
- }
- }
- }
- return false;
-}
-
export type { ProvisionToolContext } from '../code/queued';
export interface ProvisionCallbackDeps {
req: ServerRequest;
+ fileStorage: FileProvisionStorage;
agentToolContexts: Map<string, ProvisionToolContext>;
resolvePrimaryAgentId?: () => string | undefined;
/** A child routed per call keeps its own context, keyed by its execution. */
@@ -93,14 +77,12 @@ function resolveProvisionContext({
*/
export function createProvisionFilesCallback({
req,
+ fileStorage,
agentToolContexts,
resolvePrimaryAgentId,
resolveExecutionContext,
provisionToCodeEnv,
provisionToVectorDB,
- updateFile,
- updateCodeEnvRef,
- addEmbeddedEntity,
}: ProvisionCallbackDeps): (
toolNames: string[],
agentId?: string,
@@ -178,6 +160,9 @@ export function createProvisionFilesCallback({
return [];
}
+ const userId = req.user?.id;
+ if (!userId) throw new Error('File provisioning requires an authenticated user');
+
/** Chat attachments and generated code outputs stay in the user's sandbox and unscoped
* vector index; only this agent's own setup files are scoped to it. Membership decides
* that, not the record's context: a user may attach another agent's setup file to this
@@ -273,44 +258,42 @@ export function createProvisionFilesCallback({
uploads.map(async ({ file, destination: sandboxFilename }) => {
const provisioned = await shareProvisioning(
shareKey(`code:${codeRouteKey}`, file, sandboxFilename),
- async () => {
- const {
- referenceSet: refs,
- refUpdate,
- sandboxFilename: storedSandboxFilename,
- } = await provisionToCodeEnv({
- req,
+ () =>
+ withFileProvisioning({
file,
- entity_id: entityIdForFile(file),
- route: ctx.codeExecutionContext,
- sandboxFilename,
- signal,
- rateLimitBudget: codeApiRateLimitBudget,
- });
- signal?.throwIfAborted();
- /* primeCodeFiles re-reads the database and skips files without a stored
- * reference, so a tool loaded after an unpersisted upload would run against
- * an attachment it cannot see. Fail instead of answering from missing input. */
- const persisted = await persistWithRetry(
- () => updateCodeEnvRef(refUpdate),
- (error) =>
- logger.error(
- `[provisionFiles] Failed to persist code environment reference for file ${refUpdate.file_id}`,
- error,
- ),
- );
- if (!persisted) {
- throw new Error(
- `Failed to persist the code environment reference for file ${refUpdate.file_id}`,
- );
- }
- return { refs, sandboxFilename: storedSandboxFilename };
- },
+ userId,
+ storage: fileStorage,
+ destination: JSON.stringify([
+ 'code',
+ codeRouteKey,
+ entityIdForFile(file) ? 'agent' : 'user',
+ entityIdForFile(file) ?? userId,
+ null,
+ sandboxFilename,
+ ]),
+ operation: async (_claim, onUploadStart) => {
+ const { refUpdate, sandboxFilename: storedSandboxFilename } =
+ await provisionToCodeEnv({
+ req,
+ file,
+ entity_id: entityIdForFile(file),
+ route: ctx.codeExecutionContext,
+ sandboxFilename,
+ signal,
+ rateLimitBudget: codeApiRateLimitBudget,
+ onUploadStart,
+ });
+ const ref = { ...refUpdate.ref, sandboxFilename: storedSandboxFilename };
+ return {
+ value: ref,
+ commit: { code: { ...refUpdate, ref } },
+ };
+ },
+ }),
);
- const { refs: referenceSet, sandboxFilename: returnedSandboxFilename } = provisioned;
- file.metadata = { ...file.metadata, ...referenceSet };
+ file.metadata = { ...file.metadata, ...mergeCodeEnvRef(file.metadata, provisioned) };
addProvisionedFile(file, EToolResources.execute_code);
- const ref = referenceSet.codeEnvRefs?.[codeRouteKey];
+ const ref = provisioned;
if (ref?.storage_session_id && ref.file_id) {
const codeFile = {
id: ref.file_id,
@@ -319,7 +302,7 @@ export function createProvisionFilesCallback({
/* The name the sandbox stored it under: a converted image is renamed on
* upload, and telling the model the record's name sends it to a path that
* does not exist. */
- name: returnedSandboxFilename,
+ name: ref.sandboxFilename ?? file.filename,
kind: ref.kind,
} as CodeEnvFile;
const existingIndex = provisionedCodeFiles.findIndex(
@@ -354,37 +337,30 @@ export function createProvisionFilesCallback({
const queuedVectorFiles = provisionState.vectorDBFiles;
const results = await Promise.allSettled(
queuedVectorFiles.map(async (file) => {
- const result = await shareProvisioning(shareKey('search', file), async () => {
- const provisioned = await provisionToVectorDB({
- req,
+ const result = await shareProvisioning(shareKey('search', file), () =>
+ withFileProvisioning({
file,
- entity_id: entityIdForFile(file),
- signal,
- });
- signal?.throwIfAborted();
- /* The vectors are already stored, so a failed flag write costs a re-embed next
- * turn rather than this turn's results. Logged, not fatal. */
- if (provisioned.embedded && provisioned.fileUpdate) {
- const update = provisioned.fileUpdate;
- /* Vectors live under the entity that provisioned them, so the namespace is
- * recorded alongside the flag. Agents sharing a record, as a duplicate does
- * with its source, each need their own embedding, and an unscoped upload
- * lands in the user's namespace rather than in none. */
- const namespace = namespaceForFile(file);
- await persistWithRetry(
- () =>
- namespace != null
- ? addEmbeddedEntity({ file_id: update.file_id, entityId: namespace })
- : updateFile(update),
- (error) =>
- logger.error(
- `[provisionFiles] Failed to persist embedding state for file ${update.file_id}`,
- error,
- ),
- );
- }
- return provisioned;
- });
+ userId,
+ storage: fileStorage,
+ destination: JSON.stringify(['search', namespaceForFile(file)]),
+ operation: async (_claim, onUploadStart) => {
+ const provisioned = await provisionToVectorDB({
+ req,
+ file,
+ entity_id: entityIdForFile(file),
+ signal,
+ onUploadStart,
+ });
+ if (!provisioned.embedded || !provisioned.fileUpdate) {
+ throw new Error('Vector store did not confirm file provisioning');
+ }
+ return {
+ value: provisioned,
+ commit: { embedded: true, entityId: namespaceForFile(file) },
+ };
+ },
+ }),
+ );
if (result.embedded) {
file.embedded = true;
const namespace = namespaceForFile(file);
diff --git a/packages/api/src/files/provision/service.ts b/packages/api/src/files/provision/service.ts
index db8b9091e8..b742d5a0b6 100644
--- a/packages/api/src/files/provision/service.ts
+++ b/packages/api/src/files/provision/service.ts
@@ -13,6 +13,7 @@ import {
} from 'librechat-data-provider';
import type { CodeEnvRef, CodeEnvRefMap, TFile } from 'librechat-data-provider';
import type { Readable } from 'node:stream';
+import type { FileStorageUploadObserver } from '../operations';
import type { CodeExecutionContext } from '~/agents/execution';
import type { CodeEnvIdentity } from '~/files/code/identity';
import type { CodeApiRateLimitBudget } from '~/utils';
@@ -52,6 +53,7 @@ export interface ProvisionDeps {
file_id: string;
entity_id?: string;
signal?: AbortSignal;
+ onUploadStart?: FileStorageUploadObserver;
}) => Promise<{ embedded?: boolean } | undefined>;
loadAuthValues: (params: {
userId: string;
@@ -98,6 +100,7 @@ export interface ProvisionService {
sandboxFilename?: string;
signal?: AbortSignal;
rateLimitBudget?: CodeApiRateLimitBudget;
+ onUploadStart?: FileStorageUploadObserver;
}) => Promise<{
referenceSet: CodeEnvReferenceSetResult;
refUpdate: CodeEnvRefUpdate;
@@ -110,6 +113,7 @@ export interface ProvisionService {
entity_id?: string;
existingStream?: Readable;
signal?: AbortSignal;
+ onUploadStart?: FileStorageUploadObserver;
}) => Promise<{ embedded: boolean; fileUpdate: ProvisionFileUpdate | null }>;
checkSessionsAlive: (params: {
files: TFile[];
@@ -265,6 +269,7 @@ export function createProvisionService({
sandboxFilename: requestedSandboxFilename,
signal,
rateLimitBudget,
+ onUploadStart,
}: {
req: ServerRequest;
file: TFile;
@@ -273,6 +278,7 @@ export function createProvisionService({
sandboxFilename?: string;
signal?: AbortSignal;
rateLimitBudget?: CodeApiRateLimitBudget;
+ onUploadStart?: FileStorageUploadObserver;
}) {
signal?.throwIfAborted();
const { handleFileUpload: uploadCodeEnvFile } = getStrategyFunctions(FileSources.execute_code);
@@ -325,10 +331,11 @@ export function createProvisionService({
...(route?.baseUrl ? { codeApiBaseUrl: route.baseUrl } : {}),
executionProfile,
signal,
+ onUploadStart,
}),
});
- signal?.throwIfAborted();
+ /* A successful upload must reach pointer persistence even if cancellation just arrived. */
/* Merge rather than overwrite: the eager upload path persists the same shape via
* mergeCodeEnvRef, so both the legacy pointer and the route-keyed map stay in sync
* and pointers for other Code API routes survive re-provisioning. */
@@ -378,6 +385,7 @@ export function createProvisionService({
file,
entity_id,
existingStream,
+ onUploadStart,
signal,
}: {
req: ServerRequest;
@@ -385,6 +393,7 @@ export function createProvisionService({
entity_id?: string;
existingStream?: Readable;
signal?: AbortSignal;
+ onUploadStart?: FileStorageUploadObserver;
}): Promise<{ embedded: boolean; fileUpdate: { file_id: string; embedded?: boolean } | null }> {
/* Unique per attempt: two concurrent requests provisioning the same file_id would
* otherwise share one path, and the first to finish unlinks it while the second is
@@ -423,8 +432,8 @@ export function createProvisionService({
file_id: file.file_id,
entity_id,
signal,
+ onUploadStart,
});
- signal?.throwIfAborted();
const embedded = embeddingResult?.embedded ?? false;
diff --git a/packages/api/src/files/rag.spec.ts b/packages/api/src/files/rag.spec.ts
index 3aab3ba239..9120646b13 100644
--- a/packages/api/src/files/rag.spec.ts
+++ b/packages/api/src/files/rag.spec.ts
@@ -20,7 +20,7 @@ jest.mock('axios', () => ({
}));
import axios from 'axios';
-import { deleteRagFile, hasEntityEmbeddings } from './rag';
+import { deleteRagFile, hasEntityEmbeddings, createVectorFileDeleter } from './rag';
import { logger } from '@librechat/data-schemas';
import { generateShortLivedToken } from '~/crypto/jwt';
@@ -231,3 +231,48 @@ describe('hasEntityEmbeddings', () => {
expect(mockedAxios.post).not.toHaveBeenCalled();
});
});
+
+describe('createVectorFileDeleter receipts', () => {
+ beforeEach(() => jest.clearAllMocks());
+ const file = { file_id: 'retained-file', embedded: true };
+ const req = { user: { id: 'owner' } };
+ const setup = (baseUrl: string | undefined = 'https://rag.example') => {
+ const remove = createVectorFileDeleter({
+ baseUrl,
+ httpClient: axios,
+ generateToken: () => 'test-token',
+ });
+ return remove;
+ };
+
+ it('returns a receipt only after a confirmed delete', async () => {
+ mockedAxios.delete.mockResolvedValueOnce({ status: 200 });
+ expect(await setup()(req, file)).toEqual({ deleted: true });
+ expect(mockedAxios.delete).toHaveBeenCalledWith(
+ 'https://rag.example/documents',
+ expect.objectContaining({ data: ['retained-file'] }),
+ );
+ });
+
+ it('confirms an already absent vector copy', async () => {
+ mockedAxios.delete.mockRejectedValueOnce({
+ code: 'ERR_BAD_REQUEST',
+ response: { status: 404 },
+ });
+ expect(await setup()(req, file)).toEqual({ deleted: true });
+ });
+
+ it('does not issue a receipt for an unconfigured store or a non-embedded file', async () => {
+ expect(await setup('')(req, file)).toBeUndefined();
+ expect(await setup()(req, { ...file, embedded: false })).toBeUndefined();
+ });
+
+ it.each([
+ new Error('network down'),
+ { response: { status: 403 } },
+ { response: { status: 500 } },
+ ])('propagates failed requests without a deletion receipt', async (error) => {
+ mockedAxios.delete.mockRejectedValueOnce(error);
+ await expect(setup()(req, file)).rejects.toBe(error);
+ });
+});
diff --git a/packages/api/src/files/rag.ts b/packages/api/src/files/rag.ts
index a52f3aa6af..109c76766d 100644
--- a/packages/api/src/files/rag.ts
+++ b/packages/api/src/files/rag.ts
@@ -1,6 +1,8 @@
import axios from 'axios';
import { logger } from '@librechat/data-schemas';
+import type { AxiosInstance } from 'axios';
import { generateShortLivedToken } from '~/crypto/jwt';
+import { isMissingStorageError } from './storage';
interface DeleteRagFileParams {
/** The user ID. Required for authentication. If not provided, the function returns false and logs an error. */
@@ -108,3 +110,34 @@ export async function hasEntityEmbeddings({
);
return Array.isArray(data) && data.some(([chunk]) => chunk?.metadata?.user_id === entity_id);
}
+
+/** A skipped or failed request must never be reported as a removed vector copy. */
+export function createVectorFileDeleter({
+ baseUrl,
+ httpClient,
+ generateToken,
+}: {
+ baseUrl?: string;
+ httpClient: Pick<AxiosInstance, 'delete'>;
+ generateToken: (userId: string) => string;
+}): (
+ req: { user: { id: string } },
+ file: { file_id: string; embedded?: boolean },
+) => Promise<{ deleted: true } | undefined> {
+ return async (req, file) => {
+ if (!file.embedded || !baseUrl) return undefined;
+ try {
+ await httpClient.delete(`${baseUrl}/documents`, {
+ headers: {
+ Authorization: `Bearer ${generateToken(req.user.id)}`,
+ 'Content-Type': 'application/json',
+ accept: 'application/json',
+ },
+ data: [file.file_id],
+ });
+ } catch (error) {
+ if (!isMissingStorageError(error)) throw error;
+ }
+ return { deleted: true };
+ };
+}
diff --git a/packages/api/src/files/storage.spec.ts b/packages/api/src/files/storage.spec.ts
new file mode 100644
index 0000000000..6fc34687af
--- /dev/null
+++ b/packages/api/src/files/storage.spec.ts
@@ -0,0 +1,625 @@
+import path from 'node:path';
+import { tmpdir } from 'node:os';
+import { FileSources } from 'librechat-data-provider';
+import { mkdtemp, readFile, rm, unlink, writeFile } from 'node:fs/promises';
+import type { FileStorageDeletionObserver } from './dispatch';
+import { createDeleteFileWithSecondaryStorage, isMissingStorageError } from './storage';
+import { createFileProvisionRecovery } from './operations';
+import { dispatchFileDeletion } from './dispatch';
+
+const req: {
+ user: { id: string; tenantId: string };
+ config: { fileConfig: { retainedFileCleanup?: boolean } };
+} = {
+ user: { id: 'owner', tenantId: 'tenant-a' },
+ config: { fileConfig: { retainedFileCleanup: true } },
+};
+const client = { provider: 'openai' };
+const file = {
+ file_id: 'attachment',
+ user: 'owner',
+ tenantId: 'tenant-a',
+ embedded: true,
+ metadata: { codeEnvRef: { file_id: 'code-copy' } },
+};
+
+function setup(source: string = FileSources.local) {
+ const primary = jest
+ .fn()
+ .mockImplementation(
+ async (
+ _req: typeof req,
+ _file: typeof file,
+ _client?: typeof client,
+ onDelete?: FileStorageDeletionObserver,
+ ) => dispatchFileDeletion(onDelete, async () => ({ deleted: true })),
+ );
+ const vectors = jest.fn().mockResolvedValue({ deleted: true });
+ const code = jest.fn().mockResolvedValue({ deleted: true });
+ const getDeleteFile = jest.fn((secondary: string) =>
+ secondary === FileSources.vectordb ? vectors : code,
+ );
+ const logger = { warn: jest.fn() };
+ const clearFileEmbeddings = jest.fn().mockResolvedValue(true);
+ const isFileStorageClean = jest.fn().mockResolvedValue(true);
+ const fileStorage = {
+ recovery: createFileProvisionRecovery(),
+ acquireFileProvision: jest.fn().mockResolvedValue(true),
+ releaseFileProvision: jest.fn().mockResolvedValue(true),
+ commitFileProvision: jest.fn().mockResolvedValue(true),
+ acquireFileDeletion: jest.fn().mockResolvedValue({ resumed: false }),
+ releaseFileDeletion: jest.fn().mockResolvedValue(true),
+ };
+ const remove = createDeleteFileWithSecondaryStorage<typeof req, typeof file, typeof client>({
+ source,
+ deleteFile: primary,
+ getDeleteFile,
+ clearFileEmbeddings,
+ isFileStorageClean,
+ fileStorage,
+ logger,
+ });
+ return {
+ primary,
+ vectors,
+ code,
+ getDeleteFile,
+ clearFileEmbeddings,
+ isFileStorageClean,
+ fileStorage,
+ logger,
+ remove,
+ };
+}
+
+describe('createDeleteFileWithSecondaryStorage', () => {
+ beforeEach(() => {
+ file.embedded = true;
+ file.metadata = { codeEnvRef: { file_id: 'code-copy' } };
+ });
+ it('waits for both secondaries before deleting the primary with its provider client', async () => {
+ const { primary, vectors, code, remove } = setup(FileSources.openai);
+ let finishCode: () => void = () => {};
+ const pendingCode = new Promise<{ deleted: true }>((resolve) => {
+ finishCode = () => resolve({ deleted: true });
+ });
+ code.mockReturnValue(pendingCode);
+
+ const deletion = remove(req, file, client);
+ await new Promise<void>((resolve) => setImmediate(resolve));
+ expect(vectors).toHaveBeenCalledWith(req, file);
+ expect(code).toHaveBeenCalledWith(req, file);
+ expect(primary).not.toHaveBeenCalled();
+ finishCode();
+ await deletion;
+
+ expect(primary).toHaveBeenCalledTimes(1);
+ expect(primary).toHaveBeenCalledWith(req, file, client);
+ expect(file.user).toBe('owner');
+ expect(file.tenantId).toBe('tenant-a');
+ });
+
+ it.each([FileSources.openai, FileSources.azure])(
+ 'deletes an authorized provider-only assistant file from %s without a local row',
+ async (source) => {
+ const { primary, isFileStorageClean, getDeleteFile, clearFileEmbeddings, logger } =
+ setup(source);
+ const remove = createDeleteFileWithSecondaryStorage({
+ source,
+ deleteFile: primary,
+ getDeleteFile,
+ clearFileEmbeddings,
+ isFileStorageClean,
+ fileStorage: {
+ recovery: createFileProvisionRecovery(),
+ acquireFileProvision: jest.fn().mockResolvedValue(true),
+ releaseFileProvision: jest.fn().mockResolvedValue(true),
+ commitFileProvision: jest.fn().mockResolvedValue(true),
+ acquireFileDeletion: jest.fn().mockResolvedValue({ resumed: false }),
+ releaseFileDeletion: jest.fn().mockResolvedValue(true),
+ },
+ logger,
+ });
+ isFileStorageClean.mockImplementation(
+ async (snapshot) => snapshot.allowMissingFileRecord === true,
+ );
+ const providerFile = { file_id: 'provider-only', embedded: false };
+ await remove({ ...req, body: { assistant_id: 'assistant' } }, providerFile, client);
+ expect(isFileStorageClean).toHaveBeenCalledWith(
+ expect.objectContaining({
+ file_id: providerFile.file_id,
+ userId: req.user.id,
+ tenantId: req.user.tenantId,
+ allowMissingFileRecord: true,
+ }),
+ );
+ expect(primary).toHaveBeenCalledWith(expect.anything(), providerFile, client);
+ },
+ );
+
+ it.each([FileSources.local, FileSources.openai, FileSources.azure])(
+ 'does not treat a persisted %s file as provider-only',
+ async (source) => {
+ const { primary, isFileStorageClean, remove } = setup(source);
+ isFileStorageClean.mockResolvedValue(false);
+ const assistantRequest = { ...req, body: { assistant_id: 'assistant' } };
+ await expect(remove(assistantRequest, file, client)).rejects.toThrow(
+ 'File storage cleanup was not confirmed',
+ );
+ expect(isFileStorageClean).toHaveBeenCalledWith(
+ expect.not.objectContaining({ allowMissingFileRecord: true }),
+ );
+ expect(primary).not.toHaveBeenCalled();
+ },
+ );
+
+ it.each(['replacement', 'failure'])(
+ 'retains primary bytes when final code-only cleanup has a %s',
+ async (mode) => {
+ const { primary, code, isFileStorageClean, remove } = setup();
+ file.embedded = false;
+ if (mode === 'replacement') isFileStorageClean.mockResolvedValue(false);
+ else isFileStorageClean.mockRejectedValue(new Error('resource not found'));
+ await expect(remove(req, file)).rejects.toThrow('File storage cleanup was not confirmed');
+ expect(code).toHaveBeenCalledTimes(1);
+ expect(primary).not.toHaveBeenCalled();
+ },
+ );
+
+ it.each([401, 403, 500])('keeps primary bytes on HTTP %s', async (status) => {
+ const { primary, code, remove } = setup();
+ const dir = await mkdtemp(path.join(tmpdir(), 'file-delete-'));
+ const primaryPath = path.join(dir, 'attachment.png');
+ try {
+ await writeFile(primaryPath, 'readable attachment');
+ primary.mockImplementation(() => unlink(primaryPath));
+ const error = Object.assign(new Error('secondary unavailable'), { response: { status } });
+ code.mockRejectedValue(error);
+
+ await expect(remove(req, file)).rejects.toBe(error);
+ expect(primary).not.toHaveBeenCalled();
+ expect(await readFile(primaryPath, 'utf8')).toBe('readable attachment');
+ } finally {
+ await rm(dir, { recursive: true, force: true });
+ }
+ });
+
+ it('retries partial secondary cleanup without skipping the primary', async () => {
+ const { primary, vectors, code, remove } = setup();
+ const failure = new Error('code unavailable');
+ const absent = Object.assign(new Error('already removed'), {
+ code: 'ERR_BAD_REQUEST',
+ response: { status: 404 },
+ });
+ vectors.mockResolvedValueOnce({ deleted: true }).mockRejectedValueOnce(absent);
+ code.mockRejectedValueOnce(failure).mockResolvedValueOnce({ deleted: true });
+
+ await expect(remove(req, file)).rejects.toBe(failure);
+ expect(primary).not.toHaveBeenCalled();
+ await expect(remove(req, file)).resolves.toBeUndefined();
+ expect(vectors).toHaveBeenCalledTimes(1);
+ expect(code).toHaveBeenCalledTimes(2);
+ expect(primary).toHaveBeenCalledTimes(1);
+ });
+
+ it('does not let a missing secondary hide a failure in another secondary', async () => {
+ const { primary, vectors, code, remove } = setup();
+ vectors.mockRejectedValue(Object.assign(new Error('missing'), { status: 404 }));
+ const failure = new Error('permission denied');
+ code.mockRejectedValue(failure);
+
+ await expect(remove(req, file)).rejects.toBe(failure);
+ expect(primary).not.toHaveBeenCalled();
+ });
+
+ it('settles every secondary before returning a failure', async () => {
+ const { primary, vectors, code, remove } = setup();
+ const failure = new Error('vector unavailable');
+ vectors.mockRejectedValue(failure);
+ let finishCode: () => void = () => {};
+ const pendingCode = new Promise<{ deleted: true }>((resolve) => {
+ finishCode = () => resolve({ deleted: true });
+ });
+ code.mockReturnValue(pendingCode);
+ let settled = false;
+ const deletion = remove(req, file).catch((error) => {
+ settled = true;
+ return error;
+ });
+ await new Promise<void>((resolve) => setImmediate(resolve));
+ expect(settled).toBe(false);
+ expect(primary).not.toHaveBeenCalled();
+ finishCode();
+ expect(await deletion).toBe(failure);
+ });
+
+ it('retries after primary failure even when both secondaries have already gone', async () => {
+ const { primary, vectors, code, remove } = setup();
+ const failure = new Error('primary permission denied');
+ primary.mockRejectedValueOnce(failure).mockResolvedValueOnce(undefined);
+ const absent = Object.assign(new Error('absent'), { code: 'NotFound' });
+ vectors.mockResolvedValueOnce({ deleted: true }).mockRejectedValueOnce(absent);
+ code.mockResolvedValueOnce({ deleted: true }).mockRejectedValueOnce(absent);
+
+ await expect(remove(req, file)).rejects.toBe(failure);
+ await expect(remove(req, file)).resolves.toBeUndefined();
+ expect(primary).toHaveBeenCalledTimes(2);
+ });
+
+ describe.each([FileSources.s3, FileSources.cloudfront, FileSources.azure_blob])(
+ '%s mutation boundary',
+ (source) => {
+ it('releases deletion intent after preflight fails following secondary cleanup', async () => {
+ const { primary, vectors, code, fileStorage, remove } = setup(source);
+ const failure = Object.assign(new Error('preflight timed out'), { code: 'ETIMEDOUT' });
+ primary.mockRejectedValue(failure);
+ await expect(remove(req, file, client)).rejects.toBe(failure);
+ expect(vectors).toHaveBeenCalledTimes(1);
+ expect(code).toHaveBeenCalledTimes(1);
+ expect(file.embedded).toBe(false);
+ expect(primary).toHaveBeenCalledWith(req, file, client, expect.any(Function));
+ expect(fileStorage.releaseFileDeletion).toHaveBeenCalledWith(
+ expect.objectContaining({ retainDeletion: false }),
+ );
+ });
+
+ it('does not clear deletion intent inherited from an earlier attempt on preflight failure', async () => {
+ const { primary, fileStorage, remove } = setup(source);
+ fileStorage.acquireFileDeletion.mockResolvedValue({ resumed: true });
+ primary.mockRejectedValue(new Error('preflight timed out'));
+ await expect(remove(req, file)).rejects.toThrow('preflight timed out');
+ expect(fileStorage.releaseFileDeletion).toHaveBeenCalledWith(
+ expect.objectContaining({ retainDeletion: true }),
+ );
+ });
+
+ it('retains deletion intent when the dispatched delete loses its response', async () => {
+ const { primary, fileStorage, remove } = setup(source);
+ primary.mockImplementation((_req, _file, _client, onDelete) =>
+ dispatchFileDeletion(onDelete, async () => {
+ throw Object.assign(new Error('delete response lost'), { code: 'ETIMEDOUT' });
+ }),
+ );
+ await expect(remove(req, file)).rejects.toThrow('delete response lost');
+ expect(fileStorage.releaseFileDeletion).toHaveBeenCalledWith(
+ expect.objectContaining({ retainDeletion: true }),
+ );
+ });
+
+ it('reopens repair after the actual delete is definitively rejected', async () => {
+ const { primary, fileStorage, remove } = setup(source);
+ primary.mockImplementation((_req, _file, _client, onDelete) =>
+ dispatchFileDeletion(onDelete, async () => {
+ throw Object.assign(new Error('delete denied'), { statusCode: 403 });
+ }),
+ );
+ await expect(remove(req, file)).rejects.toThrow('delete denied');
+ expect(fileStorage.releaseFileDeletion).toHaveBeenCalledWith(
+ expect.objectContaining({ retainDeletion: false }),
+ );
+ });
+
+ it('retains deletion intent after a confirmed delete followed by RAG rejection', async () => {
+ const { primary, fileStorage, remove } = setup(source);
+ primary.mockImplementation(async (_req, _file, _client, onDelete) => {
+ await dispatchFileDeletion(onDelete, async () => undefined);
+ throw Object.assign(new Error('RAG denied'), { statusCode: 403 });
+ });
+ await expect(remove(req, file)).rejects.toThrow('RAG denied');
+ expect(fileStorage.releaseFileDeletion).toHaveBeenCalledWith(
+ expect.objectContaining({ retainDeletion: true }),
+ );
+ });
+ },
+ );
+
+ it('retains deletion intent when primary dispatch loses its response', async () => {
+ const { primary, fileStorage, remove } = setup();
+ primary.mockRejectedValue(Object.assign(new Error('response lost'), { code: 'ETIMEDOUT' }));
+ await expect(remove(req, file)).rejects.toThrow('response lost');
+ expect(fileStorage.releaseFileDeletion).toHaveBeenCalledWith(
+ expect.objectContaining({ retainDeletion: true }),
+ );
+ });
+
+ it.each([
+ { response: { status: 403 } },
+ { $metadata: { httpStatusCode: 403 } },
+ { statusCode: 403 },
+ { status: 403 },
+ ])('reopens repair after a definitive primary access rejection: %j', async (denial) => {
+ const { primary, fileStorage, remove } = setup();
+ primary.mockRejectedValue(Object.assign(new Error('access denied'), denial));
+ await expect(remove(req, file)).rejects.toThrow('access denied');
+ expect(fileStorage.releaseFileDeletion).toHaveBeenCalledWith(
+ expect.objectContaining({ retainDeletion: false }),
+ );
+ });
+
+ it('retains uncertainty from a second secondary instead of forgiving it as the first denial', async () => {
+ const { vectors, code, primary, fileStorage, remove } = setup();
+ vectors.mockRejectedValue(
+ Object.assign(new Error('vector denied'), { response: { status: 403 } }),
+ );
+ code.mockRejectedValue(Object.assign(new Error('code response lost'), { code: 'ETIMEDOUT' }));
+ await expect(remove(req, file)).rejects.toThrow('vector denied');
+ expect(primary).not.toHaveBeenCalled();
+ expect(fileStorage.releaseFileDeletion).toHaveBeenCalledWith(
+ expect.objectContaining({ retainDeletion: true }),
+ );
+ });
+
+ it('logs only safe metadata for an already-removed secondary', async () => {
+ const { vectors, logger, remove } = setup();
+ vectors.mockRejectedValue(
+ Object.assign(new Error('private provider message'), {
+ response: { status: 404, data: 'private provider payload' },
+ }),
+ );
+ await remove(req, file);
+ expect(logger.warn).toHaveBeenCalledWith('File storage was already missing during delete', {
+ type: 'Error',
+ status: 404,
+ });
+ });
+
+ it('accepts an absent primary only after the secondaries are cleaned up', async () => {
+ const { primary, vectors, code, remove } = setup();
+ primary.mockRejectedValue(Object.assign(new Error('absent'), { code: 'ENOENT' }));
+
+ await expect(remove(req, file)).resolves.toBeUndefined();
+ expect(vectors).toHaveBeenCalledTimes(1);
+ expect(code).toHaveBeenCalledTimes(1);
+ });
+
+ it.each([FileSources.vectordb, FileSources.execute_code])(
+ 'does not delete the primary %s source twice',
+ async (source) => {
+ const { primary, getDeleteFile, remove } = setup(source);
+ await remove(req, file, client);
+ expect(getDeleteFile).not.toHaveBeenCalledWith(source);
+ expect(getDeleteFile).toHaveBeenCalledTimes(1);
+ expect(primary).toHaveBeenCalledTimes(1);
+ },
+ );
+
+ it.each([false, true])(
+ 'reconciles confirmed primary-vector cleanup before returning (already absent: %s)',
+ async (absent) => {
+ const { primary, clearFileEmbeddings, remove } = setup(FileSources.vectordb);
+ const retained = { ...file, metadata: { ...file.metadata, embeddedEntities: ['namespace'] } };
+ if (absent)
+ primary.mockRejectedValue(Object.assign(new Error('already gone'), { status: 404 }));
+ await remove(req, retained, client);
+ expect(primary).toHaveBeenCalledTimes(1);
+ expect(clearFileEmbeddings).toHaveBeenCalledTimes(1);
+ expect(clearFileEmbeddings).toHaveBeenCalledWith({
+ file_id: retained.file_id,
+ userId: retained.user,
+ tenantId: retained.tenantId,
+ updatedAt: undefined,
+ embeddedEntities: ['namespace'],
+ });
+ expect(primary.mock.invocationCallOrder[0]).toBeLessThan(
+ clearFileEmbeddings.mock.invocationCallOrder[0],
+ );
+ expect(retained.embedded).toBe(false);
+ expect(retained.metadata).toEqual(expect.objectContaining({ searchDestination: true }));
+ expect(retained.metadata.embeddedEntities).toBeUndefined();
+ },
+ );
+
+ it.each(['conflict', 'failure'])(
+ 'reports incomplete primary-vector cleanup after reconciliation %s',
+ async (mode) => {
+ const { primary, clearFileEmbeddings, remove } = setup(FileSources.vectordb);
+ if (mode === 'conflict') clearFileEmbeddings.mockResolvedValue(false);
+ else clearFileEmbeddings.mockRejectedValue(new Error('resource not found'));
+ await expect(remove(req, file, client)).rejects.toThrow(
+ 'File embedding reconciliation failed',
+ );
+ expect(primary).toHaveBeenCalledTimes(1);
+ expect(file.embedded).toBe(true);
+ },
+ );
+
+ it('uses the codeEnvRefs map without changing the owner or provider references', async () => {
+ const { primary, code } = setup();
+ const mapped = {
+ ...file,
+ embedded: false,
+ metadata: { codeEnvRefs: { stateful: { file_id: 'stateful-copy' } } },
+ };
+ const deleteMapped = createDeleteFileWithSecondaryStorage({
+ source: FileSources.local,
+ deleteFile: primary,
+ getDeleteFile: () => code,
+ clearFileEmbeddings: jest.fn().mockResolvedValue(true),
+ isFileStorageClean: jest.fn().mockResolvedValue(true),
+ fileStorage: {
+ recovery: createFileProvisionRecovery(),
+ acquireFileProvision: jest.fn().mockResolvedValue(true),
+ releaseFileProvision: jest.fn().mockResolvedValue(true),
+ commitFileProvision: jest.fn().mockResolvedValue(true),
+ acquireFileDeletion: jest.fn().mockResolvedValue({ resumed: false }),
+ releaseFileDeletion: jest.fn().mockResolvedValue(true),
+ },
+ logger: { warn: jest.fn() },
+ });
+ await deleteMapped(req, mapped);
+ expect(code).toHaveBeenCalledWith(req, mapped);
+ expect(mapped.metadata.codeEnvRefs.stateful.file_id).toBe('stateful-copy');
+ expect(mapped.user).toBe('owner');
+ expect(mapped.tenantId).toBe('tenant-a');
+ });
+
+ it.each(['primary', 'code'])('clears vectors on %s failure', async (failing) => {
+ const { primary, vectors, code, clearFileEmbeddings, remove } = setup();
+ const retained = {
+ ...file,
+ metadata: { ...file.metadata, embeddedEntities: ['agent-a', 'agent-b'], keep: true },
+ };
+ vectors.mockResolvedValue({ deleted: true });
+ const failure = new Error('provider down');
+ (failing === 'primary' ? primary : code).mockRejectedValue(failure);
+
+ await expect(remove(req, retained)).rejects.toBe(failure);
+ expect(clearFileEmbeddings).toHaveBeenCalledWith({
+ file_id: 'attachment',
+ userId: 'owner',
+ tenantId: 'tenant-a',
+ updatedAt: undefined,
+ embeddedEntities: ['agent-a', 'agent-b'],
+ });
+ expect(retained.embedded).toBe(false);
+ expect(retained.metadata).toEqual({
+ codeEnvRef: file.metadata.codeEnvRef,
+ keep: true,
+ searchDestination: true,
+ });
+ if (failing === 'code') expect(primary).not.toHaveBeenCalled();
+ });
+
+ it('fails closed on skipped vector requests before primary deletion', async () => {
+ const { primary, vectors, clearFileEmbeddings, remove } = setup();
+ vectors.mockResolvedValueOnce(undefined).mockRejectedValueOnce(new Error('rag down'));
+ await expect(remove(req, file)).rejects.toThrow('Vector deletion was not confirmed');
+ await expect(remove(req, file)).rejects.toThrow('rag down');
+ expect(primary).not.toHaveBeenCalled();
+ expect(clearFileEmbeddings).not.toHaveBeenCalled();
+ expect(file.embedded).toBe(true);
+ });
+
+ it.each([undefined, false])(
+ 'stages embedded cleanup before reader rollout (%s)',
+ async (enabled) => {
+ const { primary, vectors, code, remove } = setup();
+ const staged = { ...req, config: { fileConfig: { retainedFileCleanup: enabled } } };
+ await expect(remove(staged, file)).rejects.toThrow('requires upgraded readers');
+ expect(primary).not.toHaveBeenCalled();
+ expect(vectors).not.toHaveBeenCalled();
+ expect(code).not.toHaveBeenCalled();
+ expect(file.embedded).toBe(true);
+ },
+ );
+
+ it('stages code-reference invalidation before marker-aware readers are deployed', async () => {
+ const { primary, code, vectors, remove } = setup();
+ const codeOnly = { ...file, embedded: false };
+ await expect(remove({ ...req, config: { fileConfig: {} } }, codeOnly)).rejects.toThrow(
+ 'requires upgraded readers',
+ );
+ expect(primary).not.toHaveBeenCalled();
+ expect(code).not.toHaveBeenCalled();
+ expect(vectors).not.toHaveBeenCalled();
+ });
+
+ it('retains primary when code cleanup was skipped', async () => {
+ const { primary, code, remove } = setup();
+ code.mockResolvedValue(undefined);
+ await expect(remove(req, file)).rejects.toThrow('Code storage deletion was not confirmed');
+ expect(primary).not.toHaveBeenCalled();
+ });
+
+ it('keeps every retry reference when both secondaries fail', async () => {
+ const { primary, vectors, code, clearFileEmbeddings, remove } = setup();
+ vectors.mockRejectedValue(new Error('rag down'));
+ code.mockRejectedValue(new Error('code down'));
+ await expect(remove(req, file)).rejects.toThrow('rag down');
+ expect(clearFileEmbeddings).not.toHaveBeenCalled();
+ expect(primary).not.toHaveBeenCalled();
+ expect(file.metadata.codeEnvRef.file_id).toBe('code-copy');
+ });
+
+ it.each(['outage', 'conflict'])('retains primary on reconciliation %s', async (failure) => {
+ const { primary, vectors, clearFileEmbeddings, remove } = setup();
+ vectors.mockResolvedValue({ deleted: true });
+ if (failure === 'outage') {
+ clearFileEmbeddings.mockRejectedValue(new Error('resource not found'));
+ } else {
+ clearFileEmbeddings.mockResolvedValue(false);
+ }
+ await expect(remove(req, file)).rejects.toThrow('File embedding reconciliation failed');
+ expect(primary).not.toHaveBeenCalled();
+ expect(file.embedded).toBe(true);
+ expect(file.metadata.codeEnvRef.file_id).toBe('code-copy');
+ });
+
+ it('retries primary deletion without probing an empty reconciled code map', async () => {
+ const { primary, getDeleteFile } = setup();
+ const remove = createDeleteFileWithSecondaryStorage({
+ source: FileSources.local,
+ deleteFile: primary,
+ getDeleteFile,
+ clearFileEmbeddings: jest.fn().mockResolvedValue(true),
+ isFileStorageClean: jest.fn().mockResolvedValue(true),
+ fileStorage: {
+ recovery: createFileProvisionRecovery(),
+ acquireFileProvision: jest.fn().mockResolvedValue(true),
+ releaseFileProvision: jest.fn().mockResolvedValue(true),
+ commitFileProvision: jest.fn().mockResolvedValue(true),
+ acquireFileDeletion: jest.fn().mockResolvedValue({ resumed: false }),
+ releaseFileDeletion: jest.fn().mockResolvedValue(true),
+ },
+ logger: { warn: jest.fn() },
+ });
+ await remove(req, {
+ ...file,
+ embedded: false,
+ metadata: { codeEnvRefs: {}, codeDestination: true },
+ });
+ expect(primary).toHaveBeenCalledTimes(1);
+ expect(getDeleteFile).not.toHaveBeenCalled();
+ });
+
+ it('only deletes primary storage for a file without secondary references', async () => {
+ const { primary, getDeleteFile } = setup();
+ const remove = createDeleteFileWithSecondaryStorage({
+ source: FileSources.local,
+ deleteFile: primary,
+ getDeleteFile,
+ clearFileEmbeddings: jest.fn().mockResolvedValue(true),
+ isFileStorageClean: jest.fn().mockResolvedValue(true),
+ fileStorage: {
+ recovery: createFileProvisionRecovery(),
+ acquireFileProvision: jest.fn().mockResolvedValue(true),
+ releaseFileProvision: jest.fn().mockResolvedValue(true),
+ commitFileProvision: jest.fn().mockResolvedValue(true),
+ acquireFileDeletion: jest.fn().mockResolvedValue({ resumed: false }),
+ releaseFileDeletion: jest.fn().mockResolvedValue(true),
+ },
+ logger: { warn: jest.fn() },
+ });
+ await remove(req, {});
+ expect(getDeleteFile).not.toHaveBeenCalled();
+ expect(primary).toHaveBeenCalledWith(req, {}, undefined);
+ });
+});
+
+describe('isMissingStorageError', () => {
+ it.each([
+ { code: 'ENOENT' },
+ { code: 'NoSuchKey' },
+ { code: 'NotFound' },
+ { code: 'ResourceNotFound' },
+ { status: 404 },
+ { statusCode: '404' },
+ { code: 'ERR_BAD_REQUEST', response: { status: 404 } },
+ new Error('blob does not exist'),
+ ])('recognizes an absent object: %j', (error) => {
+ expect(isMissingStorageError(error)).toBe(true);
+ });
+
+ it.each([
+ null,
+ undefined,
+ { response: { status: 401 } },
+ { response: { status: 403 } },
+ { response: { status: 500 } },
+ { code: 'ETIMEDOUT' },
+ { code: 'EACCES' },
+ new Error('Configuration not found'),
+ ])('does not forgive operational failures: %j', (error) => {
+ expect(isMissingStorageError(error)).toBe(false);
+ });
+});
diff --git a/packages/api/src/files/storage.ts b/packages/api/src/files/storage.ts
new file mode 100644
index 0000000000..485c1364d3
--- /dev/null
+++ b/packages/api/src/files/storage.ts
@@ -0,0 +1,271 @@
+import { FileSources, getCodeEnvRefs } from 'librechat-data-provider';
+import type { ClearFileEmbeddingsParams, FileStorageCleanupParams } from '@librechat/data-schemas';
+import type { CodeEnvReferenceSet } from 'librechat-data-provider';
+import type { FileStorageDeletionObserver } from './dispatch';
+import type { FileDeleteStorage } from './operations';
+import { withFileDeletion, isDefinitiveFileStorageFailure } from './operations';
+import { getSafeErrorMetadata } from '../utils/errors';
+
+const MISSING_STORAGE_CODES = new Set<string | number>([
+ 404,
+ '404',
+ 'ENOENT',
+ 'NoSuchKey',
+ 'NotFound',
+ 'ResourceNotFound',
+]);
+
+type StorageFile = {
+ file_id?: string;
+ user?: { toString(): string };
+ tenantId?: string;
+ updatedAt?: Date | string;
+ embedded?: boolean;
+ metadata?: Record<string, unknown>;
+};
+
+type DeleteStorageFile<TRequest, TFile, TClient = never> = (
+ req: TRequest,
+ file: TFile,
+ client?: TClient,
+ onDelete?: FileStorageDeletionObserver,
+) => Promise<unknown>;
+
+export function isMissingStorageError(error: unknown): boolean {
+ if (error == null || typeof error !== 'object') {
+ return false;
+ }
+ const response = 'response' in error ? error.response : undefined;
+ const codes = [
+ 'code' in error ? error.code : undefined,
+ 'status' in error ? error.status : undefined,
+ 'statusCode' in error ? error.statusCode : undefined,
+ response != null && typeof response === 'object' && 'status' in response
+ ? response.status
+ : undefined,
+ ];
+ if (
+ codes.some(
+ (code) =>
+ (typeof code === 'string' || typeof code === 'number') && MISSING_STORAGE_CODES.has(code),
+ )
+ ) {
+ return true;
+ }
+ return (
+ 'message' in error &&
+ typeof error.message === 'string' &&
+ /(?:file|object|blob|key|resource) (?:not found|does not exist)|no such (?:file|key)/i.test(
+ error.message,
+ )
+ );
+}
+
+function readEmbeddedEntities(value: unknown): string[] | undefined {
+ if (value == null) return undefined;
+ if (
+ !Array.isArray(value) ||
+ !value.every((entity): entity is string => typeof entity === 'string')
+ ) {
+ throw new Error('Invalid file embedding snapshot');
+ }
+ return value;
+}
+
+/** Keep the primary readable until every secondary has settled successfully or is already absent. */
+export function createDeleteFileWithSecondaryStorage<
+ TRequest extends {
+ user: { id: string; tenantId?: string };
+ config?: { fileConfig?: { retainedFileCleanup?: boolean } };
+ body?: { assistant_id?: string };
+ },
+ TFile extends StorageFile,
+ TClient,
+>({
+ source,
+ deleteFile,
+ getDeleteFile,
+ clearFileEmbeddings,
+ isFileStorageClean,
+ fileStorage,
+ logger,
+}: {
+ source: string;
+ deleteFile: DeleteStorageFile<TRequest, TFile, TClient>;
+ getDeleteFile: (source: string) => DeleteStorageFile<TRequest, TFile>;
+ clearFileEmbeddings: (data: ClearFileEmbeddingsParams) => Promise<boolean>;
+ isFileStorageClean: (data: FileStorageCleanupParams) => Promise<boolean>;
+ fileStorage: FileDeleteStorage;
+ logger: { warn: (message: string, error: unknown) => void };
+}): DeleteStorageFile<TRequest, TFile, TClient> {
+ return async (req, file, client) => {
+ const hasCodeEnvRef =
+ getCodeEnvRefs(file.metadata as CodeEnvReferenceSet | undefined).length > 0;
+ const providerOnly =
+ (source === FileSources.openai || source === FileSources.azure) &&
+ req.body?.assistant_id != null &&
+ file.user == null &&
+ file.updatedAt == null &&
+ file.embedded !== true &&
+ !hasCodeEnvRef;
+ if (
+ (file.embedded === true || hasCodeEnvRef) &&
+ req.config?.fileConfig?.retainedFileCleanup !== true
+ ) {
+ throw new Error('Retained file cleanup requires upgraded readers');
+ }
+ const deleteStorage = async (
+ primaryDeleted: (confirmed?: boolean) => void,
+ onOutcomeUnknown: () => void,
+ ): Promise<void> => {
+ const secondaries: Array<DeleteStorageFile<TRequest, TFile>> = [];
+ let vectorIndex = -1;
+ if (file.embedded === true && source !== FileSources.vectordb) {
+ vectorIndex = secondaries.length;
+ secondaries.push(getDeleteFile(FileSources.vectordb));
+ }
+ if (hasCodeEnvRef && source !== FileSources.execute_code) {
+ secondaries.push(getDeleteFile(FileSources.execute_code));
+ }
+
+ let primaryStarted = false;
+ const remove = async (
+ operation: () => Promise<unknown>,
+ primary = false,
+ ): Promise<unknown> => {
+ try {
+ return await operation();
+ } catch (error) {
+ if (!isMissingStorageError(error)) {
+ if ((!primary || primaryStarted) && !isDefinitiveFileStorageFailure(error)) {
+ onOutcomeUnknown();
+ }
+ throw error;
+ }
+ logger.warn(
+ 'File storage was already missing during delete',
+ getSafeErrorMetadata(error),
+ );
+ return { deleted: true };
+ }
+ };
+
+ const reconcileVectors = async (): Promise<void> => {
+ try {
+ if (
+ !file.file_id ||
+ !(await clearFileEmbeddings({
+ file_id: file.file_id,
+ userId: file.user?.toString() ?? req.user.id,
+ tenantId: file.tenantId,
+ updatedAt: file.updatedAt,
+ embeddedEntities: readEmbeddedEntities(file.metadata?.embeddedEntities),
+ }))
+ ) {
+ throw new Error('Embedding snapshot changed');
+ }
+ } catch (error) {
+ throw new Error('File embedding reconciliation failed', { cause: error });
+ }
+ file.embedded = false;
+ file.metadata = { ...file.metadata, searchDestination: true };
+ delete file.metadata.embeddedEntities;
+ };
+ const results = await Promise.allSettled(
+ secondaries.map((deleteSecondary) => remove(() => deleteSecondary(req, file))),
+ );
+ const vectors = results[vectorIndex];
+ if (vectorIndex >= 0 && vectors?.status === 'fulfilled') {
+ if (
+ vectors.value == null ||
+ typeof vectors.value !== 'object' ||
+ !('deleted' in vectors.value) ||
+ vectors.value.deleted !== true
+ ) {
+ throw new Error('Vector deletion was not confirmed');
+ }
+ await reconcileVectors();
+ }
+ for (const result of results) {
+ if (result.status === 'rejected') {
+ throw result.reason;
+ }
+ }
+ const code = results[vectorIndex >= 0 ? 1 : 0];
+ if (
+ hasCodeEnvRef &&
+ source !== FileSources.execute_code &&
+ code?.status === 'fulfilled' &&
+ (code.value == null ||
+ typeof code.value !== 'object' ||
+ !('deleted' in code.value) ||
+ code.value.deleted !== true)
+ ) {
+ throw new Error('Code storage deletion was not confirmed');
+ }
+ const confirmCleanup = async (): Promise<void> => {
+ if (!file.file_id) return;
+ try {
+ if (
+ !(await isFileStorageClean({
+ file_id: file.file_id,
+ userId: file.user?.toString() ?? req.user.id,
+ tenantId: providerOnly ? req.user.tenantId : file.tenantId,
+ updatedAt: file.updatedAt,
+ ...(providerOnly ? { allowMissingFileRecord: true } : {}),
+ source,
+ embedded: file.embedded,
+ embeddedEntities: readEmbeddedEntities(file.metadata?.embeddedEntities),
+ }))
+ ) {
+ throw new Error('File storage snapshot changed');
+ }
+ } catch (error) {
+ throw new Error('File storage cleanup was not confirmed', { cause: error });
+ }
+ };
+ if (source !== FileSources.execute_code) await confirmCleanup();
+ const onDelete: FileStorageDeletionObserver = (state) => {
+ if (state === undefined) {
+ primaryStarted = true;
+ primaryDeleted(false);
+ } else if (state === 'confirmed') {
+ primaryDeleted(true);
+ }
+ };
+ const marked =
+ source === FileSources.s3 ||
+ source === FileSources.cloudfront ||
+ source === FileSources.azure_blob;
+ if (!marked) onDelete();
+ const primary = await remove(
+ () => (marked ? deleteFile(req, file, client, onDelete) : deleteFile(req, file, client)),
+ true,
+ );
+ if (
+ ((source === FileSources.vectordb && file.embedded === true) ||
+ (source === FileSources.execute_code && hasCodeEnvRef)) &&
+ (primary == null ||
+ typeof primary !== 'object' ||
+ !('deleted' in primary) ||
+ primary.deleted !== true)
+ ) {
+ throw new Error('Primary secondary-store deletion was not confirmed');
+ }
+ primaryDeleted(true);
+ if (source === FileSources.vectordb && file.embedded === true) await reconcileVectors();
+ if (source === FileSources.execute_code) await confirmCleanup();
+ };
+ if (!file.file_id || providerOnly)
+ return deleteStorage(
+ () => undefined,
+ () => undefined,
+ );
+ return withFileDeletion({
+ file: { ...file, file_id: file.file_id },
+ userId: req.user.id,
+ storage: fileStorage,
+ operation: deleteStorage,
+ });
+ };
+}
diff --git a/packages/api/src/projects/resources.spec.ts b/packages/api/src/projects/resources.spec.ts
index 0eaf4f431e..265f0d7a93 100644
--- a/packages/api/src/projects/resources.spec.ts
+++ b/packages/api/src/projects/resources.spec.ts
@@ -1,14 +1,18 @@
import mongoose from 'mongoose';
-import { FileContext } from 'librechat-data-provider';
import { MongoMemoryServer } from 'mongodb-memory-server';
+import { FileContext, FileSources } from 'librechat-data-provider';
import { createModels, createMethods } from '@librechat/data-schemas';
import type { IChatProject, IMongoFile, ProjectFileRecord } from '@librechat/data-schemas';
import {
getChatProjectFileAvailability,
listChatProjectFileViews,
resolveChatProjectFiles,
+ resolveChatProjectPolicyFiles,
+ toCanonicalProjectResource,
+ toRuntimeFile,
} from './resources';
import { getChatProjectContextKey, resolveChatProjectContext } from './context';
+import { filterFilesByEndpointRuntimeConfig } from '../files/filter';
let mongoServer: InstanceType<typeof MongoMemoryServer>;
let File: mongoose.Model<IMongoFile>;
@@ -69,38 +73,120 @@ const getProjectFiles = async ({
tenantId?: string | null;
includeContent?: boolean;
}): Promise<ProjectFileRecord[]> => {
- const files = await File.find({
- file_id: { $in: fileIds },
- user: userId,
- embedded: true,
- context: FileContext.message_attachment,
- tenantId: tenantId != null && tenantId !== '' ? tenantId : null,
- })
- .select(includeContent ? {} : { text: 0 })
- .lean<IMongoFile[]>();
- return files.map((file) => ({
- _id: file._id.toString(),
- file_id: file.file_id,
- filename: file.filename,
- filepath: file.filepath,
- object: file.object,
- type: file.type,
- bytes: file.bytes,
- usage: file.usage,
- ...(file.embedded !== undefined ? { embedded: file.embedded } : {}),
- ...(file.context !== undefined ? { context: file.context } : {}),
- ...(file.expiredAt !== undefined ? { expiredAt: file.expiredAt } : {}),
- user: file.user.toString(),
- ...(file.tenantId !== undefined ? { tenantId: file.tenantId } : {}),
- ...(file.createdAt !== undefined ? { createdAt: file.createdAt } : {}),
- ...(file.updatedAt !== undefined ? { updatedAt: file.updatedAt } : {}),
- ...(file.previewRevision !== undefined ? { previewRevision: file.previewRevision } : {}),
- ...(file.status !== undefined ? { status: file.status } : {}),
- ...(file.text !== undefined ? { text: file.text } : {}),
- }));
+ return createMethods(mongoose).getProjectFiles({ fileIds, userId, tenantId, includeContent });
};
describe('ChatProject resource hydration', () => {
+ it.each([
+ { searchDestination: true, routingMimeType: 'image/heic' },
+ { routingMimeType: 'image/heic' },
+ { searchDestination: false },
+ undefined,
+ ])('keeps only routing metadata in canonical and runtime files: %p', async (metadata) => {
+ const owner = new mongoose.Types.ObjectId().toString();
+ await File.create(
+ file('converted-project-file', owner, {
+ type: 'image/jpeg',
+ metadata: { ...metadata, destinationChosen: true, embeddedEntities: ['private-agent'] },
+ }),
+ );
+ const params = {
+ project: { file_ids: ['converted-project-file'] },
+ userId: owner,
+ getProjectFiles,
+ };
+ const [record] = await getProjectFiles({ fileIds: params.project.file_ids, userId: owner });
+ const resource = toCanonicalProjectResource(record);
+ if (resource.availability !== 'ready') {
+ throw new Error('Expected a ready converted Project file');
+ }
+ const [runtime] = await resolveChatProjectFiles(params);
+ const [policy] = await resolveChatProjectPolicyFiles(params);
+ const runtimeSource = {
+ ...record,
+ metadata: {
+ ...metadata,
+ routingMimeType: metadata?.routingMimeType,
+ destinationChosen: true,
+ embeddedEntities: ['private-agent'],
+ },
+ };
+ const direct = toRuntimeFile(runtimeSource);
+ for (const result of [resource.file, runtime, policy, direct]) {
+ expect(result.type).toBe('image/jpeg');
+ expect(result.metadata).toEqual(metadata);
+ if (metadata === undefined) {
+ expect(result).not.toHaveProperty('metadata');
+ }
+ }
+ expect(runtime).not.toHaveProperty('text');
+ expect(policy.text).toBe('must never be loaded');
+ for (const allowed of ['image/heic', 'image/jpeg']) {
+ expect(
+ filterFilesByEndpointRuntimeConfig(
+ {
+ config: {},
+ fileStrategy: FileSources.local,
+ imageOutputType: 'png',
+ fileConfig: { endpoints: { openAI: { supportedMimeTypes: [allowed] } } },
+ },
+ { files: [runtime], endpoint: 'openAI' },
+ ),
+ ).toHaveLength((metadata?.routingMimeType ?? 'image/jpeg') === allowed ? 1 : 0);
+ }
+ const withoutRouting = toCanonicalProjectResource({
+ ...record,
+ metadata: {
+ ...(metadata?.searchDestination !== undefined
+ ? { searchDestination: metadata.searchDestination }
+ : {}),
+ },
+ });
+ if (metadata?.routingMimeType !== undefined) {
+ expect(resource.version).not.toBe(withoutRouting.version);
+ } else {
+ expect(resource.version).toBe(withoutRouting.version);
+ }
+ });
+
+ it.each([true, false, undefined])(
+ 'preserves vector presence without guessing (%s)',
+ (embedded) => {
+ const runtime = toRuntimeFile({
+ file_id: 'retained-project-file',
+ filename: 'notes.pdf',
+ filepath: '/files/notes.pdf',
+ object: 'file',
+ type: 'application/pdf',
+ bytes: 1024,
+ usage: 0,
+ user: 'owner',
+ embedded,
+ source: 's3',
+ metadata: { searchDestination: true },
+ });
+ expect(runtime.embedded).toBe(embedded === true);
+ expect(runtime.source).toBe('s3');
+ expect(runtime.metadata?.searchDestination).toBe(true);
+ },
+ );
+
+ it('rejects an unrecognized storage source instead of routing it to another store', () => {
+ expect(() =>
+ toRuntimeFile({
+ file_id: 'retained-project-file',
+ filename: 'notes.pdf',
+ filepath: '/files/notes.pdf',
+ object: 'file',
+ type: 'application/pdf',
+ bytes: 1024,
+ usage: 0,
+ user: 'owner',
+ source: 'unrecognized-store',
+ }),
+ ).toThrow('Invalid project file storage source');
+ });
+
it('preserves Project compatibility when signed file URLs are refreshed', async () => {
const owner = new mongoose.Types.ObjectId().toString();
const methods = { ...createMethods(mongoose), getProjectFiles };
@@ -157,6 +243,57 @@ describe('ChatProject resource hydration', () => {
expect(getChatProjectContextKey(after)).toBe(getChatProjectContextKey(before));
});
+ it('keeps cleared search destinations visible and honestly repairable', async () => {
+ const owner = new mongoose.Types.ObjectId().toString();
+ const methods = createMethods(mongoose);
+ const stored = await File.create(
+ file('repair', owner, {
+ source: 's3',
+ storageKey: 'owner/repair',
+ storageRegion: 'region',
+ metadata: { destinationChosen: true },
+ }),
+ );
+ const options = { fileIds: ['repair'], userId: owner };
+ const [before] = await methods.getProjectFiles(options);
+ const originalVersion = toCanonicalProjectResource(before).version;
+ expect(
+ await methods.clearFileEmbeddings({
+ file_id: 'repair',
+ userId: owner,
+ updatedAt: stored.updatedAt,
+ }),
+ ).toBe(true);
+ const [after] = await methods.getProjectFiles(options);
+ const resource = toCanonicalProjectResource(after);
+ expect(resource.availability).toBe('ready');
+ expect(resource.version).not.toBe(originalVersion);
+ const runtime = await resolveChatProjectFiles({
+ project: { file_ids: ['repair'] },
+ userId: owner,
+ getProjectFiles: methods.getProjectFiles,
+ });
+ expect(runtime).toEqual([
+ expect.objectContaining({
+ embedded: false,
+ metadata: { searchDestination: true },
+ source: 's3',
+ storageKey: 'owner/repair',
+ storageRegion: 'region',
+ }),
+ ]);
+ expect(
+ await listChatProjectFileViews({
+ project: { file_ids: ['repair'] },
+ userId: owner,
+ getProjectFiles: methods.getProjectFiles,
+ }),
+ ).toEqual([expect.objectContaining({ file_id: 'repair', availability: 'ready' })]);
+ const withoutIntent = toCanonicalProjectResource({ ...after, metadata: undefined });
+ expect(withoutIntent.availability).toBe('unavailable');
+ expect(withoutIntent.version).not.toBe(resource.version);
+ });
+
it('only marks canonical unscoped message attachments ready', () => {
expect(
getChatProjectFileAvailability(
diff --git a/packages/api/src/projects/resources.ts b/packages/api/src/projects/resources.ts
index a86ca00764..fb9c789306 100644
--- a/packages/api/src/projects/resources.ts
+++ b/packages/api/src/projects/resources.ts
@@ -1,5 +1,5 @@
import { createHash } from 'crypto';
-import { FileContext } from 'librechat-data-provider';
+import { FileContext, FileSources } from 'librechat-data-provider';
import type { TChatProject, TChatProjectFile, TFile } from 'librechat-data-provider';
import type { IChatProject, ProjectFileRecord } from '@librechat/data-schemas';
export type { ProjectFileRecord } from '@librechat/data-schemas';
@@ -23,13 +23,18 @@ export function getChatProjectFileAvailability(
file:
| {
embedded?: boolean;
+ metadata?: { searchDestination?: boolean };
context?: string;
expiredAt?: Date | number | string | null;
}
| null
| undefined,
): 'ready' | 'unavailable' {
- if (!file || file.embedded !== true || file.context !== FileContext.message_attachment) {
+ if (
+ !file ||
+ (file.embedded !== true && file.metadata?.searchDestination !== true) ||
+ file.context !== FileContext.message_attachment
+ ) {
return 'unavailable';
}
@@ -86,6 +91,10 @@ function canonicalResourceVersion(
file_id: file.file_id,
availability,
embedded: file.embedded ?? null,
+ searchDestination: file.metadata?.searchDestination ?? null,
+ ...(file.metadata?.routingMimeType !== undefined
+ ? { routingMimeType: file.metadata.routingMimeType }
+ : {}),
context: file.context ?? null,
expiredAt: file.expiredAt?.toISOString() ?? null,
createdAt: file.createdAt?.toISOString() ?? null,
@@ -150,9 +159,24 @@ type RuntimeFileSource = {
usage: number;
user: string | { toString(): string };
tenantId?: string;
+ embedded?: boolean;
+ metadata?: { searchDestination?: boolean; routingMimeType?: string };
+ source?: string;
+ storageKey?: string;
+ storageRegion?: string;
};
+const storageSources = new Set<string>(Object.values(FileSources));
+
+function isStorageSource(value: string): value is FileSources {
+ return storageSources.has(value);
+}
+
export function toRuntimeFile(file: RuntimeFileSource): TFile {
+ const source = file.source;
+ if (source != null && !isStorageSource(source)) {
+ throw new Error('Invalid project file storage source');
+ }
return {
file_id: file.file_id,
filename: file.filename,
@@ -161,7 +185,23 @@ export function toRuntimeFile(file: RuntimeFileSource): TFile {
type: file.type,
bytes: file.bytes,
usage: file.usage,
- embedded: true,
+ embedded: file.embedded === true,
+ ...(file.metadata?.searchDestination !== undefined ||
+ file.metadata?.routingMimeType !== undefined
+ ? {
+ metadata: {
+ ...(file.metadata.searchDestination !== undefined
+ ? { searchDestination: file.metadata.searchDestination }
+ : {}),
+ ...(file.metadata.routingMimeType !== undefined
+ ? { routingMimeType: file.metadata.routingMimeType }
+ : {}),
+ },
+ }
+ : {}),
+ ...(source != null ? { source } : {}),
+ ...(file.storageKey != null ? { storageKey: file.storageKey } : {}),
+ ...(file.storageRegion != null ? { storageRegion: file.storageRegion } : {}),
context: FileContext.message_attachment,
user: typeof file.user === 'string' ? file.user : file.user.toString(),
...(file.tenantId != null ? { tenantId: file.tenantId } : {}),
diff --git a/packages/api/src/storage/cloudfront/__tests__/crud.test.ts b/packages/api/src/storage/cloudfront/__tests__/crud.test.ts
index 8b24bc6f75..1696433863 100644
--- a/packages/api/src/storage/cloudfront/__tests__/crud.test.ts
+++ b/packages/api/src/storage/cloudfront/__tests__/crud.test.ts
@@ -504,6 +504,13 @@ describe('CloudFront CRUD', () => {
expect(mockDeleteFileFromS3).toHaveBeenCalledWith(mockReq, mockFile);
});
+ it('forwards the primary mutation observer to the S3 adapter', async () => {
+ const { deleteFileFromCloudFront } = await import('~/storage/cloudfront/crud');
+ const onDelete = jest.fn();
+ await deleteFileFromCloudFront(mockReq, mockFile, undefined, onDelete);
+ expect(mockDeleteFileFromS3).toHaveBeenCalledWith(mockReq, mockFile, undefined, onDelete);
+ });
+
it('does not create invalidation when invalidateOnDelete is false', async () => {
mockGetCloudFrontConfig.mockReturnValue(
makeConfig({ invalidateOnDelete: false, distributionId: 'E123' }),
diff --git a/packages/api/src/storage/cloudfront/crud.ts b/packages/api/src/storage/cloudfront/crud.ts
index 764c4e34ee..33d09febf2 100644
--- a/packages/api/src/storage/cloudfront/crud.ts
+++ b/packages/api/src/storage/cloudfront/crud.ts
@@ -13,6 +13,7 @@ import type {
SaveURLResult,
UploadResult,
} from '~/storage/types';
+import type { FileStorageDeletionObserver } from '~/files/dispatch';
import type { ServerRequest } from '~/types';
import {
parseS3Key,
@@ -245,10 +246,16 @@ export async function uploadFileToCloudFront(
}
/** Delete file from S3 and optionally invalidate CloudFront cache. */
-export async function deleteFileFromCloudFront(req: ServerRequest, file: TFile): Promise<void> {
+export async function deleteFileFromCloudFront(
+ req: ServerRequest,
+ file: TFile,
+ client?: unknown,
+ onDelete?: FileStorageDeletionObserver,
+): Promise<void> {
const config = getCloudFrontConfig();
- await deleteFileFromS3(req, file);
+ if (onDelete) await deleteFileFromS3(req, file, client, onDelete);
+ else await deleteFileFromS3(req, file);
if (config?.invalidateOnDelete && config.distributionId) {
try {
diff --git a/packages/api/src/storage/s3/__tests__/crud.test.ts b/packages/api/src/storage/s3/__tests__/crud.test.ts
index 1187493cfe..9e523df5f3 100644
--- a/packages/api/src/storage/s3/__tests__/crud.test.ts
+++ b/packages/api/src/storage/s3/__tests__/crud.test.ts
@@ -888,6 +888,57 @@ describe('S3 CRUD', () => {
expect(s3Mock.commandCalls(DeleteObjectCommand)).toHaveLength(1);
});
+ it('does not mark deletion when HeadObject times out', async () => {
+ const mockFile = { filepath: 'images/user123/file.jpg', user: 'user123' } as TFile;
+ const onDelete = jest.fn();
+ const failure = Object.assign(new Error('head response lost'), { code: 'ETIMEDOUT' });
+ s3Mock.on(HeadObjectCommand).rejects(failure);
+ const { deleteFileFromS3 } = await import('../crud');
+ await expect(deleteFileFromS3(mockReq, mockFile, undefined, onDelete)).rejects.toBe(failure);
+ expect(onDelete).not.toHaveBeenCalled();
+ expect(s3Mock.commandCalls(DeleteObjectCommand)).toHaveLength(0);
+ expect(deleteRagFile).not.toHaveBeenCalled();
+ });
+
+ it('marks a lost DeleteObject response without confirming deletion', async () => {
+ const mockFile = { filepath: 'images/user123/file.jpg', user: 'user123' } as TFile;
+ const onDelete = jest.fn();
+ s3Mock.on(HeadObjectCommand).resolves({});
+ const failure = Object.assign(new Error('delete response lost'), { code: 'ETIMEDOUT' });
+ s3Mock.on(DeleteObjectCommand).rejects(failure);
+ const { deleteFileFromS3 } = await import('../crud');
+ await expect(deleteFileFromS3(mockReq, mockFile, undefined, onDelete)).rejects.toBe(failure);
+ expect(onDelete.mock.calls).toEqual([[]]);
+ expect(deleteRagFile).not.toHaveBeenCalled();
+ });
+
+ it('confirms DeleteObject before a later RAG cleanup denial', async () => {
+ const mockFile = { filepath: 'images/user123/file.jpg', user: 'user123' } as TFile;
+ const onDelete = jest.fn();
+ s3Mock.on(HeadObjectCommand).resolves({});
+ const failure = Object.assign(new Error('RAG denied'), { status: 403 });
+ jest.mocked(deleteRagFile).mockImplementationOnce(async () => {
+ expect(onDelete.mock.calls).toEqual([[], ['confirmed']]);
+ throw failure;
+ });
+ const { deleteFileFromS3 } = await import('../crud');
+ await expect(deleteFileFromS3(mockReq, mockFile, undefined, onDelete)).rejects.toBe(failure);
+ expect(onDelete.mock.calls).toEqual([[], ['confirmed']]);
+ });
+
+ it('reports a definitive DeleteObject denial', async () => {
+ const mockFile = { filepath: 'images/user123/file.jpg', user: 'user123' } as TFile;
+ const onDelete = jest.fn();
+ s3Mock.on(HeadObjectCommand).resolves({});
+ const failure = Object.assign(new Error('delete denied'), {
+ $metadata: { httpStatusCode: 403 },
+ });
+ s3Mock.on(DeleteObjectCommand).rejects(failure);
+ const { deleteFileFromS3 } = await import('../crud');
+ await expect(deleteFileFromS3(mockReq, mockFile, undefined, onDelete)).rejects.toBe(failure);
+ expect(onDelete.mock.calls).toEqual([[], ['rejected']]);
+ });
+
it('uses the file owner for RAG cleanup when a different authorized user deletes', async () => {
const requesterReq = { user: { id: 'sharedUser' } } as ServerRequest;
const mockFile = {
diff --git a/packages/api/src/storage/s3/crud.ts b/packages/api/src/storage/s3/crud.ts
index ee0bf4f14c..5be7b13f5c 100644
--- a/packages/api/src/storage/s3/crud.ts
+++ b/packages/api/src/storage/s3/crud.ts
@@ -31,6 +31,7 @@ import type {
UrlBuilder,
S3FileRef,
} from '~/storage/types';
+import type { FileStorageDeletionObserver } from '~/files/dispatch';
import type { StoredFileRef } from '~/storage/path';
import type { ServerRequest } from '~/types';
import {
@@ -51,6 +52,7 @@ import {
assertPathSegment,
sanitizeContentDispositionFilename,
} from '~/storage/validation';
+import { dispatchFileDeletion } from '~/files/dispatch';
import { getSafeErrorMetadata } from '~/utils/errors';
import { initializeS3 } from '~/cdn/s3';
import { deleteRagFile } from '~/files';
@@ -717,7 +719,12 @@ export function extractKeyFromS3Url(fileUrlOrKey: string): string {
}
}
-export async function deleteFileFromS3(req: ServerRequest, file: TFile): Promise<void> {
+export async function deleteFileFromS3(
+ req: ServerRequest,
+ file: TFile,
+ _client?: unknown,
+ onDelete?: FileStorageDeletionObserver,
+): Promise<void> {
if (!req.user) {
throw new Error('[deleteFileFromS3] User not authenticated');
}
@@ -759,13 +766,14 @@ export async function deleteFileFromS3(req: ServerRequest, file: TFile): Promise
} catch (headErr) {
if ((headErr as { name?: string }).name === 'NotFound') {
logger.warn(`[deleteFileFromS3] File does not exist: ${key}`);
+ onDelete?.('confirmed');
await deleteRagFile({ userId: ownerId, file });
return;
}
throw headErr;
}
- await s3.send(new DeleteObjectCommand(params));
+ await dispatchFileDeletion(onDelete, () => s3.send(new DeleteObjectCommand(params)));
await deleteRagFile({ userId: ownerId, file });
logger.debug('[deleteFileFromS3] S3 File deletion completed');
} catch (error) {
@@ -773,6 +781,7 @@ export async function deleteFileFromS3(req: ServerRequest, file: TFile): Promise
logger.error((error as Error).stack);
if ((error as { name?: string }).name === 'NoSuchKey') {
+ onDelete?.('confirmed');
await deleteRagFile({ userId: ownerId, file });
return;
}
diff --git a/packages/api/src/types/files.ts b/packages/api/src/types/files.ts
index 83c3bfc123..509282fcf6 100644
--- a/packages/api/src/types/files.ts
+++ b/packages/api/src/types/files.ts
@@ -1,6 +1,7 @@
import type { BedrockDocumentFormat } from 'librechat-data-provider';
import type { IMongoFile } from '@librechat/data-schemas';
import type { Readable } from 'stream';
+import type { FileStorageDeletionObserver } from '~/files/dispatch';
import type { DownloadURLParams } from '~/storage/types';
import type { ServerRequest } from './http';
export interface STTService {
@@ -192,5 +193,7 @@ export interface StrategyFunctions {
user?: string;
tenantId?: string | null;
},
+ client?: unknown,
+ onDelete?: FileStorageDeletionObserver,
) => Promise<void>;
}
diff --git a/packages/api/src/user/erasure.spec.ts b/packages/api/src/user/erasure.spec.ts
index 399a989411..901dd49232 100644
--- a/packages/api/src/user/erasure.spec.ts
+++ b/packages/api/src/user/erasure.spec.ts
@@ -114,8 +114,9 @@ function createDeps() {
...methods,
openCheckpointDeletion: jest.fn(async () => checkpointDeletion),
deleteSharedLinks: deleteAllSharedLinksWithCleanup,
- deleteStoredFiles: jest.fn(async () => {
+ deleteStoredFiles: jest.fn(async (files) => {
order.push('deleteStoredFiles');
+ return { deletedFileIds: files.map((file) => file.file_id), failedFileIds: [] };
}),
invalidateMCPServerTools: jest.fn(async () => undefined),
disconnectMCPServer: jest.fn(async () => undefined),
@@ -203,6 +204,29 @@ describe('eraseUserData', () => {
expect(await count('RefreshTokenBridge', { userId: otherId.toString() })).toBe(1);
});
+ it('keeps new file retry anchors and provider credentials after concurrent provisioning', async () => {
+ const userId = new mongoose.Types.ObjectId();
+ const user = userId.toString();
+ await seedUser(userId);
+ const { deps } = createDeps();
+ deps.deleteStoredFiles = jest.fn(async (files) => {
+ await model('File').insertOne({
+ user: userId,
+ file_id: 'newly-provisioned',
+ metadata: { codeEnvRefs: { default: { file_id: 'remote-copy' } } },
+ });
+ return { deletedFileIds: files.map((file) => file.file_id), failedFileIds: [] };
+ });
+ const deleteKeys = jest.spyOn(deps, 'deleteUserKey');
+ const getFiles = jest.spyOn(deps, 'getFiles');
+ await expect(eraseUserData({ userId: user }, deps)).rejects.toThrow(
+ 'Stored file deletion incomplete',
+ );
+ expect(await count('File', { user: userId, file_id: 'newly-provisioned' })).toBe(1);
+ expect(deleteKeys).not.toHaveBeenCalled();
+ expect(getFiles).toHaveBeenNthCalledWith(2, { user }, undefined, { _id: 1 }, 1);
+ });
+
it('keeps the transaction ledger when the operator retains it', async () => {
const userId = new mongoose.Types.ObjectId();
await seedUser(userId);
@@ -452,18 +476,27 @@ describe('eraseUserData', () => {
expect(order.indexOf('deleteStoredFiles')).toBeLessThan(order.indexOf('deleteUserKey'));
});
- it('still erases the account when the storage provider fails', async () => {
+ it.each(['reported', 'thrown', 'unreported'])('keeps anchors on %s failure', async (failure) => {
const userId = new mongoose.Types.ObjectId();
- const { skillId } = await seedUser(userId);
+ await seedUser(userId);
const { deps } = createDeps();
- (deps.deleteStoredFiles as jest.Mock).mockRejectedValueOnce(new Error('provider down'));
+ const removeKeys = jest.spyOn(deps, 'deleteUserKey');
+ if (failure === 'thrown') {
+ (deps.deleteStoredFiles as jest.Mock).mockRejectedValueOnce(new Error('provider down'));
+ } else {
+ (deps.deleteStoredFiles as jest.Mock).mockResolvedValueOnce({
+ deletedFileIds: [],
+ failedFileIds: failure === 'reported' ? [`file-a-${userId}`] : [],
+ });
+ }
- await eraseUserData({ userId: userId.toString() }, deps);
+ await expect(eraseUserData({ userId: userId.toString() }, deps)).rejects.toThrow();
+ expect(await count('File', { user: userId })).toBe(2);
+ expect(removeKeys).not.toHaveBeenCalled();
+ expect(await count('MCPServer', { author: userId })).toBe(2);
+ await eraseUserData({ userId: userId.toString() }, deps);
expect(await count('File', { user: userId })).toBe(0);
- expect(await methods.getSkillById(skillId)).toBeNull();
- expect(await count('AclEntry', { principalId: userId })).toBe(0);
- await finishSkillCleanup(skillId);
});
for (const modelName of ['Agent', 'PromptGroup']) {
diff --git a/packages/api/src/user/erasure.ts b/packages/api/src/user/erasure.ts
index beb2366ff4..a1c66e5df3 100644
--- a/packages/api/src/user/erasure.ts
+++ b/packages/api/src/user/erasure.ts
@@ -3,6 +3,7 @@ import { logger } from '@librechat/data-schemas';
import type { TCheckpointerConfig, MCPOptions } from 'librechat-data-provider';
import type { AllMethods, IMongoFile } from '@librechat/data-schemas';
import type { CheckpointDeletion } from '~/agents/checkpoints/deletion';
+import type { FileDeletionOutcome } from '../files/deletion';
import type { OAuthAclSubject } from '../mcp/oauth/cleanup';
import {
cleanupDeletedMCPServerOAuthUsers,
@@ -54,7 +55,7 @@ export interface UserErasureDeps extends ErasureMethods {
) => Promise<CheckpointDeletion>;
deleteSharedLinks: (userId: string) => Promise<unknown>;
/** Removes stored file objects (local disk, S3, Firebase, vector store) before their metadata. */
- deleteStoredFiles: (files: IMongoFile[]) => Promise<unknown>;
+ deleteStoredFiles: (files: IMongoFile[]) => Promise<Required<FileDeletionOutcome>>;
/** Runtime cleanup for a removed MCP server; each hook is best effort. */
invalidateMCPServerTools?: (serverName: string) => Promise<unknown>;
disconnectMCPServer?: (serverName: string) => Promise<unknown>;
@@ -230,12 +231,15 @@ export async function eraseUserData(
await deps.deleteAllAgentQueuedTurnsForUser({ user: userObjectId });
/** Before user keys: provider-stored files need the user's own assistants credentials. */
const files = await deps.getFiles({ user: userId });
- try {
- await deps.deleteStoredFiles(files ?? []);
- } catch (error) {
- logger.error(`[eraseUserData] Failed to delete stored files for user ${userId}`, error);
+ const outcome = await deps.deleteStoredFiles(files ?? []);
+ const deleted = new Set(outcome.deletedFileIds);
+ if (outcome.failedFileIds.length > 0 || files?.some((file) => !deleted.has(file.file_id))) {
+ throw new Error('Stored file deletion incomplete');
+ }
+ await deps.deleteFiles([...deleted], undefined, { files: files ?? [], userId });
+ if ((await deps.getFiles({ user: userId }, undefined, { _id: 1 }, 1))?.length) {
+ throw new Error('Stored file deletion incomplete');
}
- await deps.deleteFiles([], userId);
await deps.deleteAllUserSessions({ userId });
deps.onSessionsDeleted?.();
await deps.deleteRefreshTokenBridges({ userId, tenantId });
diff --git a/packages/api/src/user/resolver.spec.ts b/packages/api/src/user/resolver.spec.ts
index cbf55eaeb3..46b52570e5 100644
--- a/packages/api/src/user/resolver.spec.ts
+++ b/packages/api/src/user/resolver.spec.ts
@@ -10,7 +10,7 @@ function createDeps(overrides: Partial<ErasureResolverDeps> = {}) {
getAppConfig: jest
.fn()
.mockResolvedValue({ endpoints: { agents: { checkpointer: { ttl: 5 } } } }),
- processDeleteRequest: jest.fn().mockResolvedValue(undefined),
+ processDeleteRequest: jest.fn().mockResolvedValue({ deletedFileIds: [], failedFileIds: [] }),
invalidateCachedTools: jest.fn().mockResolvedValue(undefined),
getMCPManager: jest.fn(() => ({ disconnectUserConnection })),
openCheckpointDeletion: jest.fn(),
@@ -166,6 +166,14 @@ describe('createErasureResolver', () => {
});
});
+ it('preserves storage failure receipts for the account-erasure contract', async () => {
+ const { deps } = createDeps();
+ const outcome = { deletedFileIds: [], failedFileIds: ['file-1'] };
+ (deps.processDeleteRequest as jest.Mock).mockResolvedValue(outcome);
+ const { deps: erasure } = await createErasureResolver(deps)(target);
+ expect(await erasure.deleteStoredFiles([])).toBe(outcome);
+ });
+
it('carries the target profile fields for header templates', async () => {
const { deps } = createDeps();
const { deps: erasure } = await createErasureResolver(deps)({
diff --git a/packages/api/src/user/resolver.ts b/packages/api/src/user/resolver.ts
index 8712820f12..6f7f8efe8a 100644
--- a/packages/api/src/user/resolver.ts
+++ b/packages/api/src/user/resolver.ts
@@ -2,6 +2,7 @@ import type { TCheckpointerConfig } from 'librechat-data-provider';
import type { IMongoFile } from '@librechat/data-schemas';
import type { AdminErasureContext, AdminErasureTarget } from '~/admin/users';
import type { UserErasureDeps, MCPOAuthErasureHooks } from './erasure';
+import type { FileDeletionOutcome } from '../files/deletion';
import type { MCPServerLookupOptions } from '~/mcp/types';
type ErasureBaseDeps = Omit<
@@ -54,7 +55,7 @@ export interface ErasureResolverDeps {
processDeleteRequest: (params: {
req: Record<string, unknown>;
files: ErasureFile[];
- }) => Promise<unknown>;
+ }) => Promise<Required<FileDeletionOutcome>>;
invalidateCachedTools: (params: { userId: string; serverName: string }) => Promise<unknown>;
getMCPManager: () =>
| { disconnectUserConnection: (userId: string, serverName: string) => Promise<unknown> }
diff --git a/packages/data-provider/src/file-config.spec.ts b/packages/data-provider/src/file-config.spec.ts
index 311d8e9fd9..5b0ce3e9fc 100644
--- a/packages/data-provider/src/file-config.spec.ts
+++ b/packages/data-provider/src/file-config.spec.ts
@@ -2251,3 +2251,15 @@ describe('mergeFileConfig memoization', () => {
expect(mergeFileConfig(undefined)).not.toBe(beforeStatic);
});
});
+
+describe('retained file cleanup rollout', () => {
+ it('defaults to retaining embedded files until readers are upgraded', () => {
+ expect(mergeFileConfig(undefined).retainedFileCleanup).toBe(false);
+ expect(mergeFileConfig({}).retainedFileCleanup).toBe(false);
+ });
+
+ it('accepts an explicit activation only after marker-aware reader rollout', () => {
+ expect(mergeFileConfig({ retainedFileCleanup: true }).retainedFileCleanup).toBe(true);
+ expect(mergeFileConfig({ retainedFileCleanup: false }).retainedFileCleanup).toBe(false);
+ });
+});
diff --git a/packages/data-provider/src/file-config.ts b/packages/data-provider/src/file-config.ts
index 837ff1f36f..c70e195ecb 100644
--- a/packages/data-provider/src/file-config.ts
+++ b/packages/data-provider/src/file-config.ts
@@ -566,6 +566,7 @@ const assistantsFileConfig = {
};
export const fileConfig = {
+ retainedFileCleanup: false,
endpoints: {
[EModelEndpoint.assistants]: assistantsFileConfig,
[EModelEndpoint.azureAssistants]: assistantsFileConfig,
@@ -659,6 +660,8 @@ const skillFileConfigSchema = z.object({
});
export const fileConfigSchema = z.object({
+ /** Enable only after every replica and rollback target reads retained file destinations. */
+ retainedFileCleanup: z.boolean().optional(),
endpoints: z.record(endpointFileConfigSchema).optional(),
skills: skillFileConfigSchema.optional(),
serverFileSizeLimit: z.number().min(0).optional(),
@@ -1271,6 +1274,10 @@ function buildMergedFileConfig(dynamic: DynamicFileConfig | undefined): FileConf
return mergedConfig;
}
+ if (dynamic.retainedFileCleanup !== undefined) {
+ mergedConfig.retainedFileCleanup = dynamic.retainedFileCleanup;
+ }
+
if (dynamic.defaultLLMDeliveryPath !== undefined) {
mergedConfig.defaultLLMDeliveryPath = dynamic.defaultLLMDeliveryPath;
}
diff --git a/packages/data-provider/src/resolve-llm-delivery-path.spec.ts b/packages/data-provider/src/resolve-llm-delivery-path.spec.ts
index ab3728cedd..de0c9d983c 100644
--- a/packages/data-provider/src/resolve-llm-delivery-path.spec.ts
+++ b/packages/data-provider/src/resolve-llm-delivery-path.spec.ts
@@ -10,6 +10,7 @@ import {
hasTurnFileConsumer,
isNativelyReadableText,
hasToolResourceProvisioning,
+ hasToolResourceDestination,
canToolResourceConsume,
resolveUploadDestination,
getCustomEndpointProvider,
@@ -869,6 +870,17 @@ describe('hasToolResourceProvisioning', () => {
});
});
+describe('hasToolResourceDestination', () => {
+ it('retains search eligibility without claiming vectors or another destination exist', () => {
+ const file = { embedded: false, metadata: { searchDestination: true } };
+ expect(hasToolResourceDestination(file, EToolResources.file_search)).toBe(true);
+ expect(hasToolResourceProvisioning(file, EToolResources.file_search)).toBe(false);
+ expect(hasToolResourceDestination(file, EToolResources.execute_code)).toBe(false);
+ expect(hasToolResourceDestination({}, EToolResources.file_search)).toBe(false);
+ expect(hasToolResourceDestination({ embedded: true }, EToolResources.file_search)).toBe(true);
+ });
+});
+
describe('hasInferredLLMDeliveryPath', () => {
it('re-resolves only a route upload inferred', () => {
expect(hasInferredLLMDeliveryPath({ llmDeliveryPath: 'none' })).toBe(true);
@@ -951,6 +963,22 @@ describe('resolveTurnLLMDeliveryPath', () => {
).toBe('none');
});
+ it('keeps the search route while confirmed vector cleanup awaits lazy repair', () => {
+ const file = {
+ ...routedCsv,
+ embedded: false,
+ metadata: { searchDestination: true },
+ };
+ expect(
+ resolveTurnLLMDeliveryPath({
+ file,
+ consumers: { executeCode: false, fileSearch: true },
+ endpointConfig,
+ }),
+ ).toBe('none');
+ expect(resolveTurnLLMDeliveryPath({ file, consumers: noReader, endpointConfig })).toBe('text');
+ });
+
it('delivers text when File Search is on but never received the file', () => {
/* The plain-chat File Search toggle: the upload names no destination, so nothing files it
* under a tool resource and it is never embedded. Withholding the text on the strength of
@@ -1081,3 +1109,16 @@ describe('resolveTurnLLMDeliveryPath', () => {
).toBe('none');
});
});
+
+describe('retained code destination', () => {
+ it('preserves chosen code intent without claiming the deleted copy exists', () => {
+ const file: TurnDeliveryFile = {
+ type: 'text/plain',
+ llmDeliveryPath: 'none',
+ metadata: { destinationChosen: true, codeDestination: true },
+ };
+ expect(hasToolResourceProvisioning(file, EToolResources.execute_code)).toBe(false);
+ expect(hasToolResourceDestination(file, EToolResources.execute_code)).toBe(true);
+ expect(hasToolResourceDestination(file, EToolResources.file_search)).toBe(false);
+ });
+});
diff --git a/packages/data-provider/src/resolve-llm-delivery-path.ts b/packages/data-provider/src/resolve-llm-delivery-path.ts
index 6b41d378fc..2973df9b94 100644
--- a/packages/data-provider/src/resolve-llm-delivery-path.ts
+++ b/packages/data-provider/src/resolve-llm-delivery-path.ts
@@ -364,13 +364,7 @@ export interface TurnFileConsumers {
fileSearch: boolean;
}
-/**
- * Whether the record shows this file reached a tool's own store: vectors for file search, a
- * sandbox pointer for code execution. Nothing else writes either, so their presence is proof
- * the file was provisioned and their absence proof it was not. The same evidence deferred
- * provisioning reads before queueing a file, so a turn cannot withhold content for a tool that
- * provisioning has yet to serve.
- */
+/** Whether the record shows a copy in a tool's store, not merely a chosen destination. */
export function hasToolResourceProvisioning(file: TurnDeliveryFile, toolResource: string): boolean {
if (toolResource === EToolResources.execute_code) {
return getCodeEnvRefs(file.metadata).length > 0;
@@ -378,28 +372,31 @@ export function hasToolResourceProvisioning(file: TurnDeliveryFile, toolResource
return file.embedded === true || (file.metadata?.embeddedEntities?.length ?? 0) > 0;
}
+/** A durable destination survives cleanup; presence still answers whether its copy exists. */
+export function hasToolResourceDestination(file: TurnDeliveryFile, toolResource: string): boolean {
+ return (
+ (toolResource === EToolResources.file_search && file.metadata?.searchDestination === true) ||
+ (toolResource === EToolResources.execute_code && file.metadata?.codeDestination === true) ||
+ hasToolResourceProvisioning(file, toolResource)
+ );
+}
+
/**
- * Whether a tool this turn runs can read a file of this type.
- *
- * Passing the record asks the stricter question a delivery decision needs where a tool serves
- * only what it already holds. File search reads the vector store, and an upload that named no
- * destination is filed under no tool at all, so an enabled search tool is a reader only once
- * the record shows the store holds the file. Run Code needs no copy in advance: its first call
- * uploads every attachment it can read to the sandbox, so enabling it is enough. Requiring a
- * copy there would hand the file's text to the prompt on every turn before the first code run,
- * and a turn that refuses that text never runs code, so the file would never become held.
+ * Whether an enabled tool can consume this file. Search needs a proven destination so
+ * lazy repair can restore cleared vectors. Code uploads readable attachments on first use.
*/
export function hasTurnFileConsumer(
mimeType: string,
consumers: TurnFileConsumers,
file?: TurnDeliveryFile,
): boolean {
- const searchHolds = file == null || hasToolResourceProvisioning(file, EToolResources.file_search);
+ const searchDestination =
+ file == null || hasToolResourceDestination(file, EToolResources.file_search);
return (
(consumers.executeCode && canToolResourceConsume(EToolResources.execute_code, mimeType)) ||
(consumers.fileSearch &&
canToolResourceConsume(EToolResources.file_search, mimeType) &&
- searchHolds)
+ searchDestination)
);
}
@@ -436,6 +433,10 @@ export interface TurnDeliveryFile {
destinationChosen?: boolean;
/** Vector namespaces holding this file, written as each embedding succeeds. */
embeddedEntities?: string[];
+ /** Chosen search destination, including while its vectors await repair. */
+ searchDestination?: boolean;
+ /** Chosen code destination while its remote copy awaits repair. */
+ codeDestination?: boolean;
} & CodeEnvReferenceSet)
| null;
}
diff --git a/packages/data-provider/src/types/files.ts b/packages/data-provider/src/types/files.ts
index 39650f3fa5..c304a99413 100644
--- a/packages/data-provider/src/types/files.ts
+++ b/packages/data-provider/src/types/files.ts
@@ -58,6 +58,8 @@ export type EndpointFileConfig = {
};
export type FileConfig = {
+ /** Enable after all replicas and rollback targets support retained file destinations. */
+ retainedFileCleanup?: boolean;
endpoints: {
[key: string]: EndpointFileConfig;
};
@@ -101,6 +103,8 @@ export type FileConfig = {
};
export type FileConfigInput = {
+ /** Fail closed on embedded-file deletion until marker-aware readers are deployed. */
+ retainedFileCleanup?: boolean;
endpoints?: {
[key: string]: EndpointFileConfig;
};
@@ -215,6 +219,10 @@ export type TFile = {
sourceDispatchedAt?: number;
/** Vector namespaces this file has been embedded into. */
embeddedEntities?: string[];
+ /** Search remains a destination even after its vectors are cleared. */
+ searchDestination?: boolean;
+ /** Code remains a destination after a confirmed copy deletion. */
+ codeDestination?: boolean;
/** The user named this destination, so absent ones were declined. */
destinationChosen?: boolean;
/** The type the delivery route was resolved against, when conversion changed it. */
diff --git a/packages/data-schemas/src/methods/chatProject.spec.ts b/packages/data-schemas/src/methods/chatProject.spec.ts
index cdced4efa0..54527a8fc2 100644
--- a/packages/data-schemas/src/methods/chatProject.spec.ts
+++ b/packages/data-schemas/src/methods/chatProject.spec.ts
@@ -843,6 +843,25 @@ describe('persistent Project context', () => {
expect((await methods.getChatProject(owner, id))?.file_ids).toEqual([]);
});
+ it('accepts a retained search destination without claiming its vectors exist', async () => {
+ const project = await methods.createChatProject(owner, { name: 'Repairable' });
+ await createReference('repairable', {
+ embedded: false,
+ metadata: { searchDestination: true },
+ });
+ const attached = await methods.addChatProjectFile(owner, project._id!.toString(), 'repairable');
+ expect(attached?.file_ids).toEqual(['repairable']);
+ expect((await File.findOne({ file_id: 'repairable' }).lean())?.embedded).toBe(false);
+ await createReference('foreign-repair', {
+ user: new mongoose.Types.ObjectId(otherOwner),
+ embedded: false,
+ metadata: { searchDestination: true },
+ });
+ await expect(
+ methods.addChatProjectFile(owner, project._id!.toString(), 'foreign-repair'),
+ ).rejects.toThrow('Project file unavailable');
+ });
+
it('rejects unauthorized, agent-scoped, unindexed, missing and expired references', async () => {
const project = await methods.createChatProject(owner, { name: 'Private' });
const id = project._id!.toString();
diff --git a/packages/data-schemas/src/methods/chatProject.ts b/packages/data-schemas/src/methods/chatProject.ts
index 144f24d61b..258113de56 100644
--- a/packages/data-schemas/src/methods/chatProject.ts
+++ b/packages/data-schemas/src/methods/chatProject.ts
@@ -567,9 +567,11 @@ export function createChatProjectMethods(mongoose: typeof import('mongoose')): C
file_id: fileId,
user,
tenantId: project.tenantId ?? null,
- embedded: true,
context: FileContext.message_attachment,
- $or: [{ expiredAt: null }, { expiredAt: { $gt: new Date() } }],
+ $and: [
+ { $or: [{ embedded: true }, { 'metadata.searchDestination': true }] },
+ { $or: [{ expiredAt: null }, { expiredAt: { $gt: new Date() } }] },
+ ],
})
.select('_id file_id expiresAt temp_file_id')
.lean();
diff --git a/packages/data-schemas/src/methods/file.spec.ts b/packages/data-schemas/src/methods/file.spec.ts
index d504869c9e..385b1a8d0f 100644
--- a/packages/data-schemas/src/methods/file.spec.ts
+++ b/packages/data-schemas/src/methods/file.spec.ts
@@ -1,11 +1,12 @@
import mongoose from 'mongoose';
import { v4 as uuidv4 } from 'uuid';
import { MongoMemoryServer } from 'mongodb-memory-server';
-import { EToolResources, FileContext } from 'librechat-data-provider';
+import { EToolResources, FileContext, FileSources } from 'librechat-data-provider';
+import type { CodeEnvRef } from 'librechat-data-provider';
import type { IChatProject, IMongoFile } from '~/types';
import { _resetStrictCache } from '~/models/plugins/tenantIsolation';
+import { createFileMethods, FileStorageConflictError } from './file';
import { tenantStorage, runAsSystem } from '~/config/tenantContext';
-import { createFileMethods } from './file';
import { createModels } from '~/models';
let File: mongoose.Model<unknown>;
@@ -47,6 +48,1118 @@ describe('File Methods', () => {
await File.deleteMany({});
});
+ describe('clearFileCodeEnvRefs', () => {
+ const user = new mongoose.Types.ObjectId();
+ const updatedAt = new Date('2026-10-20T00:00:00Z');
+ const defaultRef: CodeEnvRef = {
+ kind: 'user',
+ id: user.toString(),
+ storage_session_id: 'default-session',
+ file_id: 'remote-default',
+ executionProfile: 'default',
+ executionRouteKey: 'default',
+ provisionedAt: 123,
+ sandboxFilename: '$notes.csv',
+ };
+ const statefulRef: CodeEnvRef = {
+ ...defaultRef,
+ storage_session_id: 'stateful-session',
+ file_id: 'remote-stateful',
+ executionProfile: 'stateful',
+ executionRouteKey: 'stateful:a',
+ };
+ const snapshot = {
+ file_id: 'retained-code-file',
+ userId: user.toString(),
+ tenantId: 'tenant-a',
+ refs: [{ routeKey: 'default', ref: defaultRef }],
+ };
+ const metadata = {
+ codeEnvRef: defaultRef,
+ codeEnvRefs: { default: defaultRef, 'stateful:a': statefulRef },
+ embeddedEntities: ['agent-a'],
+ searchDestination: true,
+ destinationChosen: true,
+ routingMimeType: 'text/csv',
+ sourceDispatchedAt: 42,
+ toolResource: EToolResources.execute_code,
+ };
+ const seed = (overrides = {}) =>
+ File.collection.insertOne({
+ file_id: snapshot.file_id,
+ user,
+ tenantId: snapshot.tenantId,
+ context: FileContext.message_attachment,
+ conversationId: 'conversation-a',
+ filename: 'notes.csv',
+ filepath: '/uploads/notes.csv',
+ type: 'text/csv',
+ bytes: 100,
+ source: 'local',
+ llmDeliveryPath: 'none',
+ embedded: true,
+ updatedAt,
+ metadata,
+ ...overrides,
+ });
+ const read = () => File.collection.findOne({ file_id: snapshot.file_id, user });
+
+ it('removes matching pointers without changing content, intent or bookkeeping', async () => {
+ await seed({
+ createdAt: updatedAt,
+ expiresAt: new Date('2030-01-01'),
+ expiredAt: new Date('2026-10-19'),
+ deletionAttempts: 3,
+ deletionRetryAt: new Date('2026-10-23'),
+ text: 'canonical source',
+ });
+ const before = await read();
+ const update = jest.spyOn(mongoose.models.File, 'updateOne');
+ expect(await fileMethods.clearFileCodeEnvRefs(snapshot)).toBe(true);
+ expect(update).toHaveBeenCalledTimes(2);
+ const paths = ['metadata.codeEnvRefs.default', 'metadata.codeEnvRef'];
+ for (const [index, path] of paths.entries()) {
+ expect(update).toHaveBeenNthCalledWith(
+ index + 1,
+ {
+ file_id: snapshot.file_id,
+ user: snapshot.userId,
+ $and: [{ tenantId: snapshot.tenantId }],
+ $or: [expect.any(Object)],
+ },
+ { $unset: { [path]: '' }, $set: { 'metadata.codeDestination': true } },
+ { timestamps: false },
+ );
+ }
+ expect(JSON.stringify(update.mock.calls)).not.toMatch(/\$expr|\$\$REMOVE/);
+ const after = await read();
+ const { codeEnvRef: _alias, codeEnvRefs: _routes, ...intent } = metadata;
+ expect(after).toEqual({
+ ...before,
+ metadata: {
+ ...intent,
+ codeDestination: true,
+ codeEnvRefs: { 'stateful:a': statefulRef },
+ },
+ });
+ expect(await fileMethods.clearFileCodeEnvRefs(snapshot)).toBe(true);
+ expect(await read()).toEqual(after);
+ });
+
+ it.each([
+ ['kind', 'agent'],
+ ['id', 'other-owner'],
+ ['storage_session_id', 'new-session'],
+ ['file_id', 'new-remote-id'],
+ ['executionProfile', 'stateful'],
+ ['executionRouteKey', 'stateful:b'],
+ ['provisionedAt', 124],
+ ['sandboxFilename', 'new-name.csv'],
+ ['version', 2],
+ ])('preserves reprovisioned pointers when %s differs', async (field, value) => {
+ const replacement = { ...defaultRef, [field]: value };
+ await seed({
+ metadata: {
+ ...metadata,
+ codeEnvRef: replacement,
+ codeEnvRefs: { default: replacement, 'stateful:a': statefulRef },
+ },
+ });
+ const before = await read();
+ expect(await fileMethods.clearFileCodeEnvRefs(snapshot)).toBe(false);
+ expect(await read()).toEqual(before);
+ });
+
+ it.each([
+ [true, undefined],
+ [true, false],
+ [true, true],
+ [false, undefined],
+ [false, false],
+ [false, true],
+ ] as const)(
+ 'retains code intent only for a matching snapshot: %s, %s',
+ async (matches, prior) => {
+ const ref = matches ? defaultRef : { ...defaultRef, provisionedAt: 124 };
+ await seed({
+ metadata: {
+ ...metadata,
+ codeEnvRef: ref,
+ codeEnvRefs: { default: ref },
+ ...(prior === undefined ? {} : { codeDestination: prior }),
+ },
+ });
+ expect(await fileMethods.clearFileCodeEnvRefs(snapshot)).toBe(matches);
+ expect((await read())?.metadata?.codeDestination).toBe(matches ? true : prior);
+ },
+ );
+
+ it('compares absent optional fields as part of the full snapshot', async () => {
+ const replacement = { ...defaultRef };
+ delete replacement.provisionedAt;
+ await seed({
+ metadata: { codeEnvRef: replacement, codeEnvRefs: { default: replacement } },
+ });
+ const before = await read();
+ expect(await fileMethods.clearFileCodeEnvRefs(snapshot)).toBe(false);
+ expect(await read()).toEqual(before);
+ });
+
+ it.each([
+ 'executionProfile',
+ 'executionRouteKey',
+ 'provisionedAt',
+ 'sandboxFilename',
+ 'version',
+ ])('distinguishes null from absent optional snapshot fields: %s', async (field) => {
+ const absent = { ...defaultRef };
+ Reflect.deleteProperty(absent, field);
+ const explicitNull = { ...absent, [field]: null };
+ await seed({
+ metadata: { codeEnvRef: explicitNull, codeEnvRefs: { default: explicitNull } },
+ });
+ const before = await read();
+ expect(
+ await fileMethods.clearFileCodeEnvRefs({
+ ...snapshot,
+ refs: [{ routeKey: 'default', ref: absent }],
+ }),
+ ).toBe(false);
+ expect(await read()).toEqual(before);
+ await File.collection.updateOne(
+ { file_id: snapshot.file_id },
+ { $set: { metadata: { codeEnvRef: absent, codeEnvRefs: { default: absent } } } },
+ );
+ const withoutField = await read();
+ expect(
+ await fileMethods.clearFileCodeEnvRefs({
+ ...snapshot,
+ refs: [{ routeKey: 'default', ref: explicitNull }],
+ }),
+ ).toBe(false);
+ expect(await read()).toEqual(withoutField);
+ await File.collection.updateOne(
+ { file_id: snapshot.file_id },
+ {
+ $set: { metadata: { codeEnvRef: explicitNull, codeEnvRefs: { default: explicitNull } } },
+ },
+ );
+ expect(
+ await fileMethods.clearFileCodeEnvRefs({
+ ...snapshot,
+ refs: [{ routeKey: 'default', ref: explicitNull }],
+ }),
+ ).toBe(true);
+ expect((await read())?.metadata?.codeEnvRefs?.default).toBeUndefined();
+ expect((await read())?.metadata?.codeEnvRef).toBeUndefined();
+ });
+
+ it('matches every field regardless of BSON property order', async () => {
+ await seed();
+ const { file_id, ...rest } = defaultRef;
+ expect(
+ await fileMethods.clearFileCodeEnvRefs({
+ ...snapshot,
+ refs: [{ routeKey: 'default', ref: { file_id, ...rest } }],
+ }),
+ ).toBe(true);
+ const stored = await read();
+ expect(stored?.metadata?.codeEnvRef).toBeUndefined();
+ expect(stored?.metadata?.codeEnvRefs).toEqual({ 'stateful:a': statefulRef });
+ });
+
+ it('clears matching sibling routes even when another snapshot was replaced', async () => {
+ const replacement = { ...defaultRef, provisionedAt: 124 };
+ await seed({
+ metadata: {
+ ...metadata,
+ codeEnvRef: replacement,
+ codeEnvRefs: { default: replacement, 'stateful:a': statefulRef },
+ },
+ });
+ expect(
+ await fileMethods.clearFileCodeEnvRefs({
+ ...snapshot,
+ refs: [...snapshot.refs, { routeKey: 'stateful:a', ref: statefulRef }],
+ }),
+ ).toBe(false);
+ const stored = await read();
+ expect(stored?.metadata?.codeEnvRef).toEqual(replacement);
+ expect(stored?.metadata?.codeEnvRefs).toEqual({ default: replacement });
+ });
+
+ it.each([statefulRef, { ...defaultRef, provisionedAt: 124 }])(
+ 'preserves an unrelated or replaced legacy alias: %p',
+ async (alias) => {
+ await seed({ metadata: { ...metadata, codeEnvRef: alias } });
+ expect(await fileMethods.clearFileCodeEnvRefs(snapshot)).toBe(alias === statefulRef);
+ const stored = await read();
+ expect(stored?.metadata?.codeEnvRef).toEqual(alias);
+ expect(stored?.metadata?.codeEnvRefs?.default).toBeUndefined();
+ },
+ );
+
+ it('preserves routes and aliases replaced between the independent conditional writes', async () => {
+ await seed();
+ const replacement = { ...defaultRef, provisionedAt: 124 };
+ const sibling = { ...statefulRef, provisionedAt: 125 };
+ const updateOne = File.collection.updateOne.bind(File.collection);
+ jest.spyOn(File.collection, 'updateOne').mockImplementation(async (...args) => {
+ const operation = args[1];
+ if (
+ operation != null &&
+ !Array.isArray(operation) &&
+ operation.$unset?.['metadata.codeEnvRef'] !== undefined
+ ) {
+ await updateOne(
+ { file_id: snapshot.file_id },
+ {
+ $set: {
+ 'metadata.codeEnvRef': replacement,
+ 'metadata.codeEnvRefs.stateful:a': sibling,
+ },
+ },
+ );
+ }
+ return updateOne(...args);
+ });
+ expect(await fileMethods.clearFileCodeEnvRefs(snapshot)).toBe(false);
+ const stored = await read();
+ expect(stored?.metadata?.codeEnvRef).toEqual(replacement);
+ expect(stored?.metadata?.codeEnvRefs).toEqual({ 'stateful:a': sibling });
+ expect(stored?.updatedAt).toEqual(updatedAt);
+ });
+
+ it.each(['metadata.codeEnvRefs.default', 'metadata.codeEnvRef'])(
+ 'confirms route absence with a narrow owned read when only %s matches',
+ async (path) => {
+ await seed({ metadata: {} });
+ await File.collection.updateOne(
+ { file_id: snapshot.file_id },
+ { $set: { [path]: defaultRef } },
+ );
+ const find = jest.spyOn(mongoose.models.File, 'findOne');
+ expect(await fileMethods.clearFileCodeEnvRefs(snapshot)).toBe(true);
+ expect(find).toHaveBeenCalledTimes(1);
+ expect(find).toHaveBeenCalledWith(
+ {
+ file_id: snapshot.file_id,
+ user: snapshot.userId,
+ $and: [{ tenantId: snapshot.tenantId }],
+ },
+ { _id: 0, 'metadata.codeEnvRef': 1, 'metadata.codeEnvRefs': 1 },
+ );
+ expect((await read())?.metadata?.codeDestination).toBe(true);
+ },
+ );
+
+ it('confirms route absence when every conditional write misses', async () => {
+ await seed({ metadata: {} });
+ const find = jest.spyOn(mongoose.models.File, 'findOne');
+ expect(await fileMethods.clearFileCodeEnvRefs(snapshot)).toBe(true);
+ expect(find).toHaveBeenCalledTimes(1);
+ expect(find).toHaveBeenCalledWith(
+ {
+ file_id: snapshot.file_id,
+ user: snapshot.userId,
+ $and: [{ tenantId: snapshot.tenantId }],
+ },
+ { _id: 0, 'metadata.codeEnvRef': 1, 'metadata.codeEnvRefs': 1 },
+ );
+ expect((await read())?.metadata).toEqual({});
+ });
+
+ it.each([false, true])(
+ 'keeps generated files hydratable with legacy-only refs: %s',
+ async (legacyOnly) => {
+ await seed({
+ context: FileContext.execute_code,
+ metadata: {
+ codeEnvRef: defaultRef,
+ ...(legacyOnly ? {} : { codeEnvRefs: { default: defaultRef } }),
+ },
+ });
+ expect(await fileMethods.clearFileCodeEnvRefs(snapshot)).toBe(true);
+ const stored = await read();
+ expect(stored?.metadata?.codeEnvRef).toBeUndefined();
+ expect(stored?.metadata?.codeEnvRefs).toEqual(legacyOnly ? undefined : {});
+ const generated = await fileMethods.getCodeGeneratedFiles(
+ 'conversation-a',
+ [snapshot.file_id],
+ snapshot,
+ );
+ expect(generated.map((file) => file.file_id)).toEqual([snapshot.file_id]);
+ },
+ );
+
+ it.each([FileContext.execute_code, FileContext.message_attachment])(
+ 'hydrates marker-only code intent without crossing owner, tenant, conversation or thread: %s',
+ async (context) => {
+ await runAsSystem(() =>
+ File.create({
+ file_id: snapshot.file_id,
+ user,
+ tenantId: snapshot.tenantId,
+ conversationId: 'conversation-a',
+ context,
+ filename: 'notes.csv',
+ filepath: '/uploads/notes.csv',
+ type: 'text/csv',
+ bytes: 100,
+ llmDeliveryPath: 'none',
+ metadata: { codeDestination: true, destinationChosen: true },
+ }),
+ );
+ const hydrate = (ids: string[], scope = snapshot, conversationId = 'conversation-a') =>
+ context === FileContext.execute_code
+ ? fileMethods.getCodeGeneratedFiles(conversationId, ids, scope)
+ : fileMethods.getUserCodeFiles(ids, scope);
+ const files = await hydrate([snapshot.file_id]);
+ expect(files.map((file) => file.file_id)).toEqual([snapshot.file_id]);
+ expect(files[0].metadata?.codeDestination).toBe(true);
+ expect(files[0].metadata?.codeEnvRef).toBeUndefined();
+ expect(files[0].metadata?.codeEnvRefs).toBeUndefined();
+ expect(await hydrate(['not-in-thread'])).toEqual([]);
+ expect(
+ await hydrate([snapshot.file_id], {
+ ...snapshot,
+ userId: new mongoose.Types.ObjectId().toString(),
+ }),
+ ).toEqual([]);
+ expect(await hydrate([snapshot.file_id], { ...snapshot, tenantId: 'tenant-b' })).toEqual(
+ [],
+ );
+ if (context === FileContext.execute_code) {
+ expect(await hydrate([snapshot.file_id], snapshot, 'other-conversation')).toEqual([]);
+ } else {
+ const deferred = await fileMethods.getDeferredProvisionFiles(
+ [snapshot.file_id],
+ snapshot,
+ { code: true },
+ );
+ expect(deferred.map((file) => file.file_id)).toEqual([snapshot.file_id]);
+ }
+ await File.updateOne(
+ { file_id: snapshot.file_id },
+ { $set: { 'metadata.codeDestination': false } },
+ );
+ expect(await hydrate([snapshot.file_id])).toEqual([]);
+ await File.updateOne(
+ { file_id: snapshot.file_id },
+ { $unset: { 'metadata.codeDestination': '' } },
+ );
+ expect(await hydrate([snapshot.file_id])).toEqual([]);
+ },
+ );
+
+ it.each([
+ {},
+ { executionProfile: 'default' as const },
+ { executionProfile: 'stateful' as const },
+ ])('clears legacy-only refs and queues the missing route: %p', async (routeFields) => {
+ const ref: CodeEnvRef = {
+ kind: 'user',
+ id: user.toString(),
+ storage_session_id: 'legacy-session',
+ file_id: 'legacy-remote',
+ ...routeFields,
+ };
+ await seed({ metadata: { ...metadata, codeEnvRef: ref, codeEnvRefs: undefined } });
+ const routeKey = ref.executionProfile ?? 'default';
+ expect(
+ await fileMethods.getDeferredProvisionFiles([snapshot.file_id], snapshot, {
+ code: true,
+ codeRouteKey: routeKey,
+ }),
+ ).toEqual([]);
+ expect(
+ await fileMethods.clearFileCodeEnvRefs({ ...snapshot, refs: [{ routeKey, ref }] }),
+ ).toBe(true);
+ const deferred = await fileMethods.getDeferredProvisionFiles([snapshot.file_id], snapshot, {
+ code: true,
+ codeRouteKey: routeKey,
+ });
+ expect(deferred.map((file) => file.file_id)).toEqual([snapshot.file_id]);
+ expect(deferred[0].llmDeliveryPath).toBe('none');
+ expect(deferred[0].metadata?.destinationChosen).toBe(true);
+ });
+
+ it('persists each copy without mutating input or invalidating the embedding snapshot', async () => {
+ await seed();
+ const input = structuredClone(snapshot);
+ expect(await fileMethods.clearFileCodeEnvRefs(snapshot)).toBe(true);
+ expect(
+ await fileMethods.clearFileCodeEnvRefs({
+ ...snapshot,
+ refs: [{ routeKey: 'stateful:a', ref: statefulRef }],
+ }),
+ ).toBe(true);
+ expect(snapshot).toEqual(input);
+ expect(
+ await fileMethods.clearFileEmbeddings({
+ ...snapshot,
+ updatedAt,
+ embeddedEntities: metadata.embeddedEntities,
+ }),
+ ).toBe(true);
+ const stored = await read();
+ expect(stored?.updatedAt).toEqual(updatedAt);
+ expect(stored?.embedded).toBe(false);
+ expect(stored?.metadata?.codeEnvRefs).toEqual({});
+ });
+
+ it.each([
+ { userId: new mongoose.Types.ObjectId().toString() },
+ { tenantId: 'tenant-b' },
+ { tenantId: undefined },
+ ])('returns false without changing an unauthorized file: %p', async (scope) => {
+ await seed();
+ const before = await read();
+ expect(await fileMethods.clearFileCodeEnvRefs({ ...snapshot, ...scope })).toBe(false);
+ expect(await read()).toEqual(before);
+ });
+
+ it.each([false, true])('scopes legacy missing/null tenants explicitly: %s', async (missing) => {
+ const foreign = await seed();
+ const local = await seed({ tenantId: null });
+ if (missing) {
+ await File.collection.updateOne({ _id: local.insertedId }, { $unset: { tenantId: '' } });
+ }
+ expect(await fileMethods.clearFileCodeEnvRefs({ ...snapshot, tenantId: undefined })).toBe(
+ true,
+ );
+ const records = await File.collection.find({ file_id: snapshot.file_id }).toArray();
+ expect(records.find((record) => record._id.equals(foreign.insertedId))?.metadata).toEqual(
+ metadata,
+ );
+ expect(
+ records.find((record) => record._id.equals(local.insertedId))?.metadata?.codeEnvRefs,
+ ).toEqual({ 'stateful:a': statefulRef });
+ });
+
+ it.each([{ tenantId: 'tenant-b' }, { tenantId: undefined }])(
+ 'does not let tenant middleware replace the caller scope: %p',
+ async (scope) => {
+ await seed();
+ const before = await read();
+ const cleared = await tenantStorage.run({ tenantId: snapshot.tenantId }, () =>
+ fileMethods.clearFileCodeEnvRefs({ ...snapshot, ...scope }),
+ );
+ expect(cleared).toBe(false);
+ expect(await read()).toEqual(before);
+ },
+ );
+
+ it('confirms no code refs remain for empty or already-cleared snapshots', async () => {
+ expect(await fileMethods.clearFileCodeEnvRefs(snapshot)).toBe(false);
+ await seed({ metadata: {} });
+ expect(await fileMethods.clearFileCodeEnvRefs({ ...snapshot, refs: [] })).toBe(true);
+ expect(await fileMethods.clearFileCodeEnvRefs(snapshot)).toBe(true);
+ expect(
+ await fileMethods.clearFileCodeEnvRefs({ ...snapshot, file_id: 'missing', refs: [] }),
+ ).toBe(false);
+ });
+
+ it.each(['metadata.codeEnvRef', 'metadata.codeEnvRefs.default'])(
+ 'rejects a primary-code-only replacement at %s',
+ async (path) => {
+ await seed({
+ source: 'execute_code',
+ context: FileContext.execute_code,
+ embedded: false,
+ metadata: {},
+ });
+ const replacement = { ...defaultRef, provisionedAt: 124 };
+ await File.collection.updateOne(
+ { file_id: snapshot.file_id },
+ { $set: { [path]: replacement } },
+ );
+ const before = await read();
+ expect(await fileMethods.clearFileCodeEnvRefs(snapshot)).toBe(false);
+ expect(await read()).toEqual(before);
+ },
+ );
+
+ it.each([false, true])(
+ 'returns false when only one route or alias snapshot matches: %s',
+ async (aliasMatches) => {
+ const replacement = { ...defaultRef, provisionedAt: 124 };
+ await seed({
+ metadata: {
+ codeEnvRef: aliasMatches ? defaultRef : replacement,
+ codeEnvRefs: { default: aliasMatches ? replacement : defaultRef },
+ },
+ });
+ expect(await fileMethods.clearFileCodeEnvRefs(snapshot)).toBe(false);
+ const stored = await read();
+ expect(stored?.metadata?.codeEnvRef).toEqual(aliasMatches ? undefined : replacement);
+ expect(stored?.metadata?.codeEnvRefs).toEqual(aliasMatches ? { default: replacement } : {});
+ expect(stored?.updatedAt).toEqual(updatedAt);
+ },
+ );
+
+ it.each([defaultRef, statefulRef])(
+ 'rejects an empty snapshot while any canonical or legacy code ref remains: %p',
+ async (ref) => {
+ await seed({ metadata: { codeEnvRef: ref } });
+ expect(await fileMethods.clearFileCodeEnvRefs({ ...snapshot, refs: [] })).toBe(false);
+ await File.collection.updateOne(
+ { file_id: snapshot.file_id },
+ { $set: { metadata: { codeEnvRefs: { unknown: ref } } } },
+ );
+ expect(await fileMethods.clearFileCodeEnvRefs({ ...snapshot, refs: [] })).toBe(false);
+ },
+ );
+
+ it('propagates the final metadata read failure after writes settle', async () => {
+ await seed();
+ jest.spyOn(mongoose.models.File, 'findOne').mockImplementationOnce(() => {
+ throw new Error('metadata read unavailable');
+ });
+ await expect(fileMethods.clearFileCodeEnvRefs(snapshot)).rejects.toThrow(
+ 'metadata read unavailable',
+ );
+ expect((await read())?.metadata?.codeEnvRef).toBeUndefined();
+ expect((await read())?.metadata?.codeEnvRefs).toEqual({ 'stateful:a': statefulRef });
+ });
+
+ it.each(['', 'stateful.invalid', '$stateful'])(
+ 'rejects invalid route keys: %s',
+ async (routeKey) => {
+ await seed();
+ const before = await read();
+ await expect(
+ fileMethods.clearFileCodeEnvRefs({
+ ...snapshot,
+ refs: [...snapshot.refs, { routeKey, ref: statefulRef }],
+ }),
+ ).rejects.toThrow('Invalid code environment route key');
+ expect(await read()).toEqual(before);
+ },
+ );
+
+ it('settles sibling writes before propagating a persistence failure', async () => {
+ await seed();
+ let finishWrite: () => void = () => undefined;
+ const pendingWrite = new Promise<void>((resolve) => {
+ finishWrite = resolve;
+ });
+ const failure = new Error('mongo unavailable');
+ const updateOne = File.collection.updateOne.bind(File.collection);
+ jest.spyOn(File.collection, 'updateOne').mockImplementation(async (...args) => {
+ const operation = args[1];
+ if (
+ operation != null &&
+ !Array.isArray(operation) &&
+ operation.$unset?.['metadata.codeEnvRefs.default'] !== undefined
+ ) {
+ throw failure;
+ }
+ await pendingWrite;
+ return updateOne(...args);
+ });
+ const find = jest.spyOn(mongoose.models.File, 'findOne');
+ let returned = false;
+ const clearing = fileMethods.clearFileCodeEnvRefs(snapshot).catch((error: Error) => {
+ returned = true;
+ return error;
+ });
+ await new Promise<void>((resolve) => setImmediate(resolve));
+ try {
+ expect(returned).toBe(false);
+ } finally {
+ finishWrite();
+ }
+ expect(await clearing).toBe(failure);
+ expect(find).not.toHaveBeenCalled();
+ const stored = await read();
+ expect(stored?.metadata?.codeEnvRef).toBeUndefined();
+ expect(stored?.metadata?.codeEnvRefs?.default).toEqual(defaultRef);
+ expect(stored?.metadata?.codeDestination).toBe(true);
+ });
+
+ it('propagates persistence failures', async () => {
+ await seed();
+ const before = await read();
+ jest.spyOn(mongoose.models.File, 'updateOne').mockImplementation(() => {
+ throw new Error('mongo unavailable');
+ });
+ await expect(fileMethods.clearFileCodeEnvRefs(snapshot)).rejects.toThrow('mongo unavailable');
+ expect(await read()).toEqual(before);
+ });
+ });
+
+ describe('clearFileEmbeddings', () => {
+ const user = new mongoose.Types.ObjectId();
+ const updatedAt = new Date('2026-10-20T00:00:00Z');
+ const snapshot = {
+ file_id: 'retained-file',
+ userId: user.toString(),
+ tenantId: 'tenant-a',
+ updatedAt,
+ embeddedEntities: ['agent-a', 'agent-b'],
+ };
+ const seed = (overrides = {}) =>
+ File.collection.insertOne({
+ file_id: snapshot.file_id,
+ user,
+ tenantId: snapshot.tenantId,
+ updatedAt,
+ embedded: true,
+ metadata: {
+ embeddedEntities: snapshot.embeddedEntities,
+ codeEnvRef: { file_id: 'still-needed' },
+ destinationChosen: true,
+ },
+ ...overrides,
+ });
+
+ it('preserves code references, TTL and retry bookkeeping', async () => {
+ const expiredAt = new Date('2026-10-19');
+ await seed({ expiredAt, deletionAttempts: 3 });
+ expect(await fileMethods.clearFileEmbeddings(snapshot)).toBe(true);
+ expect(await fileMethods.clearFileEmbeddings(snapshot)).toBe(true);
+ const stored = await File.collection.findOne({ file_id: snapshot.file_id });
+ expect(stored?.embedded).toBe(false);
+ expect(stored?.metadata).toEqual({
+ codeEnvRef: { file_id: 'still-needed' },
+ destinationChosen: true,
+ searchDestination: true,
+ });
+ expect(stored?.updatedAt).toEqual(updatedAt);
+ expect(stored?.expiredAt).toEqual(expiredAt);
+ expect(stored?.deletionAttempts).toBe(3);
+ });
+
+ it.each([true, false, undefined])(
+ 'retains search intent independently of the chooser marker %s',
+ async (destinationChosen) => {
+ await seed({
+ metadata: {
+ embeddedEntities: snapshot.embeddedEntities,
+ ...(destinationChosen !== undefined ? { destinationChosen } : {}),
+ },
+ });
+ expect(await fileMethods.clearFileEmbeddings(snapshot)).toBe(true);
+ const files = await fileMethods.getFiles({ file_id: snapshot.file_id });
+ expect(files?.[0]?.embedded).toBe(false);
+ expect(files?.[0]?.metadata?.searchDestination).toBe(true);
+ expect(files?.[0]?.metadata?.embeddedEntities).toBeUndefined();
+ expect(files?.[0]?.metadata?.destinationChosen).toBe(destinationChosen);
+ },
+ );
+
+ it('persists the optional intent through the file schema and ordinary reads', async () => {
+ await File.create({
+ file_id: snapshot.file_id,
+ user,
+ filename: 'notes.pdf',
+ filepath: '/uploads/notes.pdf',
+ type: 'application/pdf',
+ bytes: 1024,
+ embedded: false,
+ metadata: { searchDestination: true },
+ });
+ const files = await fileMethods.getFiles({ file_id: snapshot.file_id });
+ expect(files?.[0]?.metadata?.searchDestination).toBe(true);
+ expect(files?.[0]?.metadata?.destinationChosen).toBeUndefined();
+ });
+
+ it('does not change a concurrent embedding or a reused file record', async () => {
+ await seed({ updatedAt: new Date(updatedAt.getTime() + 1) });
+ expect(await fileMethods.clearFileEmbeddings(snapshot)).toBe(false);
+ const stored = await File.collection.findOne({ file_id: snapshot.file_id });
+ expect(stored?.embedded).toBe(true);
+ expect(stored?.metadata?.embeddedEntities).toEqual(snapshot.embeddedEntities);
+ });
+
+ it('does not discard a newly recorded namespace even when its timestamp matches', async () => {
+ await seed({ metadata: { embeddedEntities: ['agent-a', 'agent-b', 'agent-c'] } });
+ expect(await fileMethods.clearFileEmbeddings(snapshot)).toBe(false);
+ const stored = await File.collection.findOne({ file_id: snapshot.file_id });
+ expect(stored?.metadata?.embeddedEntities).toHaveLength(3);
+ });
+
+ it('matches the owner and tenant rather than a bare file id', async () => {
+ const first = await seed();
+ await seed({ user: new mongoose.Types.ObjectId() });
+ await seed({ tenantId: 'tenant-b' });
+ await seed({ tenantId: undefined });
+ expect(await fileMethods.clearFileEmbeddings(snapshot)).toBe(true);
+ const records = await File.collection.find({ file_id: snapshot.file_id }).toArray();
+ const clearedIds = records
+ .filter((record) => record.embedded === false)
+ .map((record) => record._id);
+ expect(clearedIds).toEqual([first.insertedId]);
+ });
+
+ it.each([null, undefined])(
+ 'clears only the untenanted row during system reconciliation: %p',
+ async (tenantId) => {
+ const foreign = await seed();
+ const local = await seed({ tenantId });
+ const before = await File.collection.findOne({ _id: foreign.insertedId });
+ expect(
+ await runAsSystem(() => fileMethods.clearFileEmbeddings({ ...snapshot, tenantId })),
+ ).toBe(true);
+ expect(await File.collection.findOne({ _id: foreign.insertedId })).toEqual(before);
+ const stored = await File.collection.findOne({ _id: local.insertedId });
+ expect(stored?.embedded).toBe(false);
+ expect(stored?.metadata?.embeddedEntities).toBeUndefined();
+ expect(stored?.metadata?.searchDestination).toBe(true);
+ expect(stored?.updatedAt).toEqual(updatedAt);
+ },
+ );
+
+ it.each([null, undefined, 'tenant-b'])(
+ 'does not clear a tenant row for a mismatched system scope: %p',
+ async (tenantId) => {
+ const seeded = await seed();
+ const before = await File.collection.findOne({ _id: seeded.insertedId });
+ expect(
+ await runAsSystem(() => fileMethods.clearFileEmbeddings({ ...snapshot, tenantId })),
+ ).toBe(false);
+ expect(await File.collection.findOne({ _id: seeded.insertedId })).toEqual(before);
+ },
+ );
+
+ it.each([null, undefined, 'tenant-b'])(
+ 'does not let embedding tenant middleware replace the caller scope: %p',
+ async (tenantId) => {
+ const seeded = await seed();
+ const before = await File.collection.findOne({ _id: seeded.insertedId });
+ expect(
+ await tenantStorage.run({ tenantId: snapshot.tenantId }, () =>
+ fileMethods.clearFileEmbeddings({ ...snapshot, tenantId }),
+ ),
+ ).toBe(false);
+ expect(await File.collection.findOne({ _id: seeded.insertedId })).toEqual(before);
+ },
+ );
+
+ it('clears legacy vector flags without an entity list', async () => {
+ await seed({ metadata: { destinationChosen: true } });
+ const legacy = { ...snapshot, embeddedEntities: undefined };
+ expect(await fileMethods.clearFileEmbeddings(legacy)).toBe(true);
+ });
+
+ it('retains search intent on markerless legacy records without metadata', async () => {
+ await File.collection.insertOne({
+ file_id: snapshot.file_id,
+ user,
+ tenantId: snapshot.tenantId,
+ updatedAt,
+ embedded: true,
+ });
+ expect(
+ await fileMethods.clearFileEmbeddings({ ...snapshot, embeddedEntities: undefined }),
+ ).toBe(true);
+ const stored = await File.collection.findOne({ file_id: snapshot.file_id });
+ expect(stored?.embedded).toBe(false);
+ expect(stored?.metadata).toEqual({ searchDestination: true });
+ });
+
+ it('propagates persistence failures instead of claiming reconciliation', async () => {
+ await seed();
+ const update = jest.spyOn(mongoose.models.File, 'updateOne').mockImplementationOnce(() => {
+ throw new Error('mongo unavailable');
+ });
+ try {
+ await expect(fileMethods.clearFileEmbeddings(snapshot)).rejects.toThrow(
+ 'mongo unavailable',
+ );
+ } finally {
+ update.mockRestore();
+ }
+ expect((await File.collection.findOne({ file_id: snapshot.file_id }))?.embedded).toBe(true);
+ });
+ });
+
+ describe('final file storage snapshot', () => {
+ const user = new mongoose.Types.ObjectId();
+ const updatedAt = new Date('2026-10-20');
+ const snapshot = {
+ file_id: 'cleanup-race',
+ user,
+ tenantId: 'tenant-a',
+ updatedAt,
+ source: 'local',
+ };
+ const options = { files: [snapshot], userId: user.toString() };
+ const check = { ...snapshot, userId: user.toString() };
+ const seed = (overrides = {}) => File.collection.insertOne({ ...snapshot, ...overrides });
+
+ it('confirms a cleared snapshot and removes only its matching owned row', async () => {
+ await seed({ embedded: false, metadata: { codeEnvRefs: {}, codeDestination: true } });
+ await seed({ tenantId: 'tenant-b' });
+ expect(await fileMethods.isFileStorageClean(check)).toBe(true);
+ expect(await fileMethods.deleteFiles([snapshot.file_id], undefined, options)).toEqual({
+ deletedCount: 1,
+ deletedFileIds: [snapshot.file_id],
+ failedFileIds: [],
+ });
+ expect(await File.collection.countDocuments({ file_id: snapshot.file_id })).toBe(1);
+ expect(await fileMethods.isFileStorageClean(check)).toBe(false);
+ expect(await fileMethods.deleteFiles([snapshot.file_id], undefined, options)).toEqual({
+ deletedCount: 0,
+ deletedFileIds: [snapshot.file_id],
+ failedFileIds: [],
+ });
+ });
+
+ it.each([
+ { metadata: { codeEnvRefs: { default: { file_id: 'replacement' } } } },
+ { metadata: { codeEnvRef: { file_id: 'replacement' } } },
+ { embedded: true, metadata: { embeddedEntities: ['new-namespace'] } },
+ { updatedAt: new Date(updatedAt.getTime() + 1) },
+ ])(
+ 'retains a reprovisioned row and reports incomplete metadata deletion: %p',
+ async (change) => {
+ await seed();
+ expect(await fileMethods.isFileStorageClean(check)).toBe(true);
+ await File.collection.updateOne({ file_id: snapshot.file_id }, { $set: change });
+ const before = await File.collection.findOne({ file_id: snapshot.file_id });
+ expect(await fileMethods.isFileStorageClean(check)).toBe(false);
+ await expect(
+ fileMethods.deleteFiles([snapshot.file_id], undefined, options),
+ ).resolves.toEqual({
+ deletedCount: 0,
+ deletedFileIds: [],
+ failedFileIds: [snapshot.file_id],
+ });
+ expect(await File.collection.findOne({ file_id: snapshot.file_id })).toEqual(before);
+ },
+ );
+
+ it('does not delete a copy attached between the metadata read and conditional delete', async () => {
+ await seed();
+ const deleteMany = File.collection.deleteMany.bind(File.collection);
+ jest
+ .spyOn(File.collection, 'deleteMany')
+ .mockImplementationOnce(async (filter, deleteOptions) => {
+ await File.collection.updateOne(
+ { file_id: snapshot.file_id },
+ { $set: { 'metadata.codeEnvRefs.default': { file_id: 'late-copy' } } },
+ );
+ return deleteMany(filter, deleteOptions);
+ });
+ await expect(
+ fileMethods.deleteFiles([snapshot.file_id], undefined, options),
+ ).resolves.toEqual({
+ deletedCount: 0,
+ deletedFileIds: [],
+ failedFileIds: [snapshot.file_id],
+ });
+ const stored = await File.collection.findOne({ file_id: snapshot.file_id });
+ expect(stored?.metadata?.codeEnvRefs?.default?.file_id).toBe('late-copy');
+ });
+
+ it('returns both receipt arrays without touching the database for an empty guarded batch', async () => {
+ const find = jest.spyOn(mongoose.models.File, 'find');
+ const remove = jest.spyOn(mongoose.models.File, 'deleteMany');
+ expect(await fileMethods.deleteFiles([], undefined, options)).toEqual({
+ deletedCount: 0,
+ deletedFileIds: [],
+ failedFileIds: [],
+ });
+ expect(find).not.toHaveBeenCalled();
+ expect(remove).not.toHaveBeenCalled();
+ });
+
+ it('detaches only confirmed successes after a mixed conditional delete in system scope', async () => {
+ const successful = { ...snapshot, file_id: 'cleanup-success' };
+ await seed();
+ await seed(successful);
+ const ChatProject = mongoose.models.ChatProject as mongoose.Model<IChatProject>;
+ const ids = [snapshot.file_id, successful.file_id];
+ const project = await ChatProject.collection.insertOne({
+ user: user.toString(),
+ tenantId: snapshot.tenantId,
+ file_ids: ids,
+ contextRevision: 4,
+ });
+ const otherTenant = await ChatProject.collection.insertOne({
+ user: user.toString(),
+ tenantId: 'tenant-b',
+ file_ids: ids,
+ contextRevision: 4,
+ });
+ const deleteMany = File.collection.deleteMany.bind(File.collection);
+ jest
+ .spyOn(File.collection, 'deleteMany')
+ .mockImplementationOnce(async (filter, deleteOptions) => {
+ await File.collection.updateOne(
+ { file_id: snapshot.file_id },
+ { $set: { 'metadata.codeEnvRefs.default': { file_id: 'late-copy' } } },
+ );
+ return deleteMany(filter, deleteOptions);
+ });
+ expect(
+ await runAsSystem(() =>
+ fileMethods.deleteFiles(ids, undefined, { ...options, files: [snapshot, successful] }),
+ ),
+ ).toEqual({
+ deletedCount: 1,
+ deletedFileIds: [successful.file_id],
+ failedFileIds: [snapshot.file_id],
+ });
+ expect(await File.collection.findOne({ file_id: successful.file_id })).toBeNull();
+ expect(
+ (await File.collection.findOne({ file_id: snapshot.file_id }))?.metadata?.codeEnvRefs,
+ ).toEqual({ default: { file_id: 'late-copy' } });
+ expect(await ChatProject.collection.findOne({ _id: project.insertedId })).toMatchObject({
+ file_ids: [snapshot.file_id],
+ contextRevision: 5,
+ });
+ expect(await ChatProject.collection.findOne({ _id: otherTenant.insertedId })).toMatchObject({
+ file_ids: ids,
+ contextRevision: 4,
+ });
+ });
+
+ it.each([undefined, null, 'tenant-a'])(
+ 'detaches already-absent input only from its snapshot owner and tenant: %s',
+ async (tenantId) => {
+ const ChatProject = mongoose.models.ChatProject as mongoose.Model<IChatProject>;
+ const project = await ChatProject.collection.insertOne({
+ user: user.toString(),
+ ...(tenantId === undefined ? {} : { tenantId }),
+ file_ids: [snapshot.file_id],
+ contextRevision: 1,
+ });
+ const otherTenant = await ChatProject.collection.insertOne({
+ user: user.toString(),
+ tenantId: 'tenant-b',
+ file_ids: [snapshot.file_id],
+ contextRevision: 1,
+ });
+ const otherOwner = await ChatProject.collection.insertOne({
+ user: new mongoose.Types.ObjectId().toString(),
+ tenantId,
+ file_ids: [snapshot.file_id],
+ contextRevision: 1,
+ });
+ expect(
+ await runAsSystem(() =>
+ fileMethods.deleteFiles([snapshot.file_id], undefined, {
+ userId: new mongoose.Types.ObjectId().toString(),
+ files: [{ ...snapshot, tenantId }],
+ }),
+ ),
+ ).toEqual({ deletedCount: 0, deletedFileIds: [snapshot.file_id], failedFileIds: [] });
+ expect(await ChatProject.collection.findOne({ _id: project.insertedId })).toMatchObject({
+ file_ids: [],
+ contextRevision: 2,
+ });
+ for (const inserted of [otherTenant, otherOwner]) {
+ expect(await ChatProject.collection.findOne({ _id: inserted.insertedId })).toMatchObject({
+ file_ids: [snapshot.file_id],
+ contextRevision: 1,
+ });
+ }
+ },
+ );
+
+ it('does not delete a replacement row that was absent from the candidate read', async () => {
+ await seed();
+ const deleteMany = File.collection.deleteMany.bind(File.collection);
+ jest
+ .spyOn(File.collection, 'deleteMany')
+ .mockImplementationOnce(async (filter, deleteOptions) => {
+ await deleteMany({ file_id: snapshot.file_id });
+ await seed();
+ return deleteMany(filter, deleteOptions);
+ });
+ expect(await fileMethods.deleteFiles([snapshot.file_id], undefined, options)).toEqual({
+ deletedCount: 0,
+ deletedFileIds: [],
+ failedFileIds: [snapshot.file_id],
+ });
+ expect(await File.collection.countDocuments({ file_id: snapshot.file_id })).toBe(1);
+ });
+
+ it('propagates operational database failures rather than issuing a receipt', async () => {
+ jest
+ .spyOn(File.collection, 'deleteMany')
+ .mockRejectedValueOnce(new Error('mongo unavailable'));
+ await expect(fileMethods.deleteFiles([snapshot.file_id], undefined, options)).rejects.toThrow(
+ 'mongo unavailable',
+ );
+ });
+
+ it('returns the successful receipt when best-effort project cleanup fails', async () => {
+ await seed();
+ jest.spyOn(mongoose.models.ChatProject, 'updateMany').mockImplementationOnce(() => {
+ throw new Error('cleanup failed');
+ });
+ expect(await fileMethods.deleteFiles([snapshot.file_id], undefined, options)).toEqual({
+ deletedCount: 1,
+ deletedFileIds: [snapshot.file_id],
+ failedFileIds: [],
+ });
+ expect(await File.collection.findOne({ file_id: snapshot.file_id })).toBeNull();
+ });
+
+ it('rejects an unknown id before attempting deletion', async () => {
+ const remove = jest.spyOn(mongoose.models.File, 'deleteMany');
+ await expect(fileMethods.deleteFiles(['unknown'], undefined, options)).rejects.toThrow(
+ 'Missing file deletion snapshot',
+ );
+ expect(remove).not.toHaveBeenCalled();
+ });
+
+ it.each([FileSources.openai, FileSources.azure])(
+ 'permits an authorized missing %s record but never a changed local snapshot',
+ async (source) => {
+ const providerOnly = { ...check, source, allowMissingFileRecord: true };
+ expect(await fileMethods.isFileStorageClean({ ...check, source })).toBe(false);
+ expect(await fileMethods.isFileStorageClean(providerOnly)).toBe(true);
+ await seed({ source, metadata: { codeEnvRefs: { default: { file_id: 'new-copy' } } } });
+ expect(await fileMethods.isFileStorageClean(providerOnly)).toBe(false);
+ await File.collection.updateOne(
+ { file_id: snapshot.file_id },
+ { $unset: { metadata: '' }, $set: { updatedAt: new Date(updatedAt.getTime() + 1) } },
+ );
+ expect(await fileMethods.isFileStorageClean(providerOnly)).toBe(false);
+ },
+ );
+
+ it('does not allow missing local records through the provider-only exception', async () => {
+ expect(await fileMethods.isFileStorageClean({ ...check, allowMissingFileRecord: true })).toBe(
+ false,
+ );
+ });
+
+ it('accepts primary vector cleanup only for its original namespace snapshot', async () => {
+ const vector = {
+ ...snapshot,
+ source: 'vectordb',
+ embedded: true,
+ metadata: { embeddedEntities: ['original'] },
+ };
+ await seed(vector);
+ const primary = {
+ ...check,
+ source: vector.source,
+ embedded: true,
+ embeddedEntities: ['original'],
+ };
+ expect(await fileMethods.isFileStorageClean(primary)).toBe(true);
+ await File.collection.updateOne(
+ { file_id: snapshot.file_id },
+ { $set: { 'metadata.embeddedEntities': ['replacement'] } },
+ );
+ expect(await fileMethods.isFileStorageClean(primary)).toBe(false);
+ await expect(
+ fileMethods.deleteFiles([snapshot.file_id], undefined, { ...options, files: [vector] }),
+ ).resolves.toEqual({
+ deletedCount: 0,
+ deletedFileIds: [],
+ failedFileIds: [snapshot.file_id],
+ });
+ });
+ });
+
describe('adoptLegacySearchEmbeddings', () => {
const Agent = () => mongoose.models.Agent as mongoose.Model<unknown>;
const author = new mongoose.Types.ObjectId();
@@ -336,6 +1449,28 @@ describe('File Methods', () => {
});
describe('getAvailableProjectFiles', () => {
+ it('retains the embedded picker index and adds a distinct search-intent partial index', () => {
+ const keys = { user: 1, tenantId: 1, context: 1, _id: -1 };
+ expect(File.schema.indexes()).toEqual(
+ expect.arrayContaining([
+ [
+ keys,
+ expect.objectContaining({
+ name: 'project_file_picker',
+ partialFilterExpression: { embedded: true },
+ }),
+ ],
+ [
+ keys,
+ expect.objectContaining({
+ name: 'project_file_picker_search_destination',
+ partialFilterExpression: { 'metadata.searchDestination': true },
+ }),
+ ],
+ ]),
+ );
+ });
+
it('treats an empty tenant id as no tenant, matching project file reads', async () => {
const userId = new mongoose.Types.ObjectId().toString();
await File.create({
@@ -444,6 +1579,87 @@ describe('File Methods', () => {
});
describe('getProjectFiles', () => {
+ it.each([
+ { searchDestination: true, routingMimeType: 'image/heic' },
+ { routingMimeType: 'image/heic' },
+ { searchDestination: false },
+ undefined,
+ ])('preserves only routing metadata across project reads: %p', async (metadata) => {
+ const userId = new mongoose.Types.ObjectId().toString();
+ await File.create({
+ file_id: 'converted-project-file',
+ user: userId,
+ tenantId: 'tenant-a',
+ embedded: true,
+ context: FileContext.message_attachment,
+ filename: 'converted.jpeg',
+ filepath: '/uploads/converted.jpeg',
+ type: 'image/jpeg',
+ bytes: 1,
+ text: 'private extracted content',
+ metadata: { ...metadata, destinationChosen: true, embeddedEntities: ['private-agent'] },
+ });
+ const options = { fileIds: ['converted-project-file'], userId, tenantId: 'tenant-a' };
+ const [projectFile] = await fileMethods.getProjectFiles(options);
+ const [withContent] = await fileMethods.getProjectFiles({ ...options, includeContent: true });
+ const available = await fileMethods.getAvailableProjectFiles(options);
+ for (const record of [projectFile, withContent, available.files[0]]) {
+ expect(record.type).toBe('image/jpeg');
+ expect(record.metadata).toEqual(metadata);
+ if (metadata === undefined) {
+ expect(record).not.toHaveProperty('metadata');
+ }
+ }
+ expect(projectFile).not.toHaveProperty('text');
+ expect(available.files[0]).not.toHaveProperty('text');
+ expect(withContent.text).toBe('private extracted content');
+ });
+
+ it('hydrates and lists retained search intent without widening owner, tenant or context scope', async () => {
+ const userId = new mongoose.Types.ObjectId().toString();
+ const base = {
+ user: userId,
+ tenantId: 'tenant-a',
+ embedded: false,
+ metadata: { searchDestination: true },
+ context: FileContext.message_attachment,
+ filename: 'repair.txt',
+ filepath: '/uploads/repair.txt',
+ type: 'text/plain',
+ bytes: 1,
+ source: 's3',
+ storageKey: 'owner/repair',
+ storageRegion: 'region',
+ };
+ await File.create([
+ { ...base, file_id: 'repair' },
+ { ...base, file_id: 'foreign', user: new mongoose.Types.ObjectId().toString() },
+ { ...base, file_id: 'other-tenant', tenantId: 'tenant-b' },
+ { ...base, file_id: 'agent', context: FileContext.agents },
+ { ...base, file_id: 'declined', metadata: { destinationChosen: true } },
+ ]);
+ const files = await fileMethods.getProjectFiles({
+ fileIds: ['repair', 'foreign', 'other-tenant', 'agent', 'declined'],
+ userId,
+ tenantId: 'tenant-a',
+ });
+ expect(files).toEqual([
+ expect.objectContaining({
+ file_id: 'repair',
+ embedded: false,
+ metadata: { searchDestination: true },
+ source: 's3',
+ storageKey: 'owner/repair',
+ storageRegion: 'region',
+ }),
+ ]);
+ const available = await fileMethods.getAvailableProjectFiles({
+ userId,
+ tenantId: 'tenant-a',
+ });
+ expect(available.files.map((file) => file.file_id)).toEqual(['repair']);
+ });
+
it('enforces owner scope and only includes content when requested', async () => {
const userId = new mongoose.Types.ObjectId().toString();
await File.create([
@@ -499,6 +1715,110 @@ describe('File Methods', () => {
});
describe('claimCodeFile', () => {
+ const fencedStates = [
+ { deletionPending: true },
+ { deleting: 'deletion-owner' },
+ { provisioning: ['provision-owner'] },
+ ];
+
+ it.each(fencedStates)('rejects a fenced filename claim: %j', async (storageOperations) => {
+ await File.init();
+ const input = {
+ filename: 'fenced.csv',
+ conversationId: 'fenced-conversation',
+ file_id: 'fenced-file',
+ user: new mongoose.Types.ObjectId().toString(),
+ tenantId: 'tenant-a',
+ sourceDispatchedAt: 100,
+ };
+ await fileMethods.claimCodeFile(input);
+ await File.updateOne({ file_id: input.file_id }, { $set: { storageOperations } });
+ const before = await File.findOne({ file_id: input.file_id }).lean();
+
+ await expect(
+ fileMethods.claimCodeFile({
+ ...input,
+ file_id: 'replacement-file',
+ sourceDispatchedAt: 200,
+ }),
+ ).rejects.toMatchObject({
+ name: 'FileStorageConflictError',
+ code: 'FILE_STORAGE_CONFLICT',
+ });
+ expect(await File.findOne({ file_id: input.file_id }).lean()).toEqual(before);
+ expect(await File.countDocuments()).toBe(1);
+
+ const independent = await fileMethods.claimCodeFile({
+ ...input,
+ filename: 'independent.csv',
+ file_id: 'independent-file',
+ });
+ expect(independent.file_id).toBe('independent-file');
+ });
+
+ it.each(fencedStates)(
+ 'rejects commits after a storage fence is acquired: %j',
+ async (storageOperations) => {
+ await File.init();
+ const input = {
+ filename: 'late-fence.csv',
+ conversationId: 'late-fence-conversation',
+ file_id: 'late-fence-file',
+ user: new mongoose.Types.ObjectId().toString(),
+ tenantId: 'tenant-a',
+ sourceDispatchedAt: 100,
+ };
+ await fileMethods.claimCodeFile(input);
+ await File.updateOne({ file_id: input.file_id }, { $set: { storageOperations } });
+ const before = await File.findOne({ file_id: input.file_id }).lean();
+ const output = {
+ ...input,
+ context: FileContext.execute_code,
+ filepath: '/uploads/new.csv',
+ type: 'text/csv',
+ bytes: 20,
+ source: FileSources.local,
+ metadata: { sourceDispatchedAt: 200 },
+ };
+
+ await expect(fileMethods.commitCodeFile(output)).rejects.toBeInstanceOf(
+ FileStorageConflictError,
+ );
+ expect(await fileMethods.commitCodeFile(output, 200)).toBe(false);
+ await expect(
+ fileMethods.createFile(
+ { ...output, user: new mongoose.Types.ObjectId(input.user) },
+ true,
+ ),
+ ).rejects.toBeInstanceOf(FileStorageConflictError);
+ expect(await File.findOne({ file_id: input.file_id }).lean()).toEqual(before);
+ expect(await File.countDocuments()).toBe(1);
+ },
+ );
+
+ it('reuses active rows with empty provisioning without changing their metadata', async () => {
+ const input = {
+ filename: 'active.csv',
+ conversationId: 'active-conversation',
+ file_id: 'active-file',
+ user: new mongoose.Types.ObjectId().toString(),
+ sourceDispatchedAt: 100,
+ };
+ await fileMethods.claimCodeFile(input);
+ await File.updateOne(
+ { file_id: input.file_id },
+ { $set: { storageOperations: { provisioning: [], deletionPending: false } } },
+ );
+ const before = await File.findOne({ file_id: input.file_id }).lean();
+ const claimed = await fileMethods.claimCodeFile({
+ ...input,
+ file_id: 'unused-file',
+ sourceDispatchedAt: 200,
+ });
+ expect(claimed.file_id).toBe(input.file_id);
+ expect(await File.findOne({ file_id: input.file_id }).lean()).toEqual(before);
+ });
+
it('claims code output files independently per tenant', async () => {
const userId = new mongoose.Types.ObjectId().toString();
diff --git a/packages/data-schemas/src/methods/file.ts b/packages/data-schemas/src/methods/file.ts
index dc68912799..c467de3347 100644
--- a/packages/data-schemas/src/methods/file.ts
+++ b/packages/data-schemas/src/methods/file.ts
@@ -1,6 +1,6 @@
import { Types } from 'mongoose';
import { createHash } from 'crypto';
-import { EToolResources, FileContext, FileSources } from 'librechat-data-provider';
+import { EToolResources, FileContext, FileSources, getCodeEnvRefs } from 'librechat-data-provider';
import type { CodeEnvRef, TFile } from 'librechat-data-provider';
import type { FilterQuery, SortOrder, Model } from 'mongoose';
import type {
@@ -11,6 +11,10 @@ import type {
CodeFileCommitData,
RunArtifactRunScope,
PublishRunArtifactInput,
+ ClearFileCodeEnvRefsParams,
+ FileStorageCleanupParams,
+ DeleteFilesOptions,
+ DeleteFilesResult,
} from '~/types/file';
import type { IChatProjectDocument } from '~/types';
import { tenantSafeBulkWrite } from '~/utils/tenantBulkWrite';
@@ -24,6 +28,14 @@ export type LegacySearchFileRef = {
tenantId?: string;
};
+export type ClearFileEmbeddingsParams = {
+ file_id: string;
+ userId: string;
+ tenantId?: string | null;
+ updatedAt?: Date | string;
+ embeddedEntities?: string[];
+};
+
export const DEFAULT_AVAILABLE_PROJECT_FILES_LIMIT = 20;
export const MAX_AVAILABLE_PROJECT_FILES_LIMIT = 50;
@@ -39,6 +51,7 @@ export type AvailableProjectFileRecord = {
bytes: number;
usage: number;
embedded?: boolean;
+ metadata?: { searchDestination?: boolean; routingMimeType?: string };
context?: string;
user: string;
tenantId?: string;
@@ -55,6 +68,7 @@ type AvailableProjectFileRecordSource = Pick<
| 'bytes'
| 'usage'
| 'embedded'
+ | 'metadata'
| 'context'
| 'user'
| 'tenantId'
@@ -73,6 +87,19 @@ function normalizeAvailableProjectFileRecord(
bytes: file.bytes,
usage: file.usage,
...(file.embedded !== undefined ? { embedded: file.embedded } : {}),
+ ...(file.metadata?.searchDestination !== undefined ||
+ file.metadata?.routingMimeType !== undefined
+ ? {
+ metadata: {
+ ...(file.metadata.searchDestination !== undefined
+ ? { searchDestination: file.metadata.searchDestination }
+ : {}),
+ ...(file.metadata.routingMimeType !== undefined
+ ? { routingMimeType: file.metadata.routingMimeType }
+ : {}),
+ },
+ }
+ : {}),
...(file.context !== undefined ? { context: file.context } : {}),
user: file.user.toString(),
...(file.tenantId !== undefined ? { tenantId: file.tenantId } : {}),
@@ -102,6 +129,10 @@ export type ProjectFileRecord = {
bytes: number;
usage: number;
embedded?: boolean;
+ metadata?: { searchDestination?: boolean; routingMimeType?: string };
+ source?: string;
+ storageKey?: string;
+ storageRegion?: string;
context?: string;
expiredAt?: Date | null;
user: string;
@@ -124,6 +155,10 @@ type ProjectFileRecordSource = Pick<
| 'bytes'
| 'usage'
| 'embedded'
+ | 'metadata'
+ | 'source'
+ | 'storageKey'
+ | 'storageRegion'
| 'context'
| 'expiredAt'
| 'user'
@@ -146,6 +181,22 @@ function normalizeProjectFileRecord(file: ProjectFileRecordSource): ProjectFileR
bytes: file.bytes,
usage: file.usage,
...(file.embedded !== undefined ? { embedded: file.embedded } : {}),
+ ...(file.metadata?.searchDestination !== undefined ||
+ file.metadata?.routingMimeType !== undefined
+ ? {
+ metadata: {
+ ...(file.metadata.searchDestination !== undefined
+ ? { searchDestination: file.metadata.searchDestination }
+ : {}),
+ ...(file.metadata.routingMimeType !== undefined
+ ? { routingMimeType: file.metadata.routingMimeType }
+ : {}),
+ },
+ }
+ : {}),
+ ...(file.source !== undefined ? { source: file.source } : {}),
+ ...(file.storageKey !== undefined ? { storageKey: file.storageKey } : {}),
+ ...(file.storageRegion !== undefined ? { storageRegion: file.storageRegion } : {}),
...(file.context !== undefined ? { context: file.context } : {}),
...(file.expiredAt !== undefined ? { expiredAt: file.expiredAt } : {}),
user: file.user.toString(),
@@ -189,6 +240,30 @@ export type ExpiredFileQueryOptions = {
now?: Date;
};
+export class FileStorageConflictError extends Error {
+ readonly code: string = 'FILE_STORAGE_CONFLICT';
+
+ constructor() {
+ super('The file has a conflicting storage operation');
+ this.name = 'FileStorageConflictError';
+ }
+}
+
+function codeFileStorageFilter(): FilterQuery<IMongoFile> {
+ return {
+ 'storageOperations.deletionPending': { $ne: true },
+ 'storageOperations.deleting': { $exists: false },
+ 'storageOperations.provisioning.0': { $exists: false },
+ };
+}
+
+function rethrowCodeFileStorageConflict(error: unknown): never {
+ if (error instanceof Error && 'code' in error && error.code === 11000) {
+ throw new FileStorageConflictError();
+ }
+ throw error;
+}
+
function assertValidCodeEnvironmentRouteKey(routeKey: string): void {
if (routeKey.length === 0 || routeKey.includes('.') || routeKey.startsWith('$')) {
throw new Error(`Invalid code environment route key "${routeKey}"`);
@@ -213,6 +288,26 @@ function withOwnerScope<T extends FilterQuery<IMongoFile>>(
return scopedFilter;
}
+/** Re-provisioned copies keep their parent row and erasure retry anchor. */
+function fileStorageCleanupFilter(data: FileStorageCleanupParams): FilterQuery<IMongoFile> {
+ return {
+ file_id: data.file_id,
+ user: data.userId,
+ $and: [{ tenantId: data.tenantId ?? null }],
+ updatedAt: data.updatedAt ?? { $exists: false },
+ 'metadata.codeEnvRef': null,
+ 'metadata.codeEnvRefs': { $in: [null, {}] },
+ 'storageOperations.provisioning.0': { $exists: false },
+ 'storageOperations.deleting': { $exists: false },
+ ...(data.source === FileSources.vectordb
+ ? {
+ embedded: data.embedded ?? { $ne: true },
+ 'metadata.embeddedEntities': data.embeddedEntities ?? null,
+ }
+ : { embedded: { $ne: true } }),
+ };
+}
+
type PlainFileDocument = Omit<TFile, 'user' | '_id'> & {
user: IMongoFile['user'];
_id?: IMongoFile['_id'];
@@ -351,6 +446,9 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
ref: CodeEnvRef;
legacyRef?: CodeEnvRef;
}) => Promise<IMongoFile | null>;
+ clearFileCodeEnvRefs: (data: ClearFileCodeEnvRefsParams) => Promise<boolean>;
+ clearFileEmbeddings: (data: ClearFileEmbeddingsParams) => Promise<boolean>;
+ isFileStorageClean: (data: FileStorageCleanupParams) => Promise<boolean>;
addFileEmbeddedEntity: (data: {
file_id: string;
entityId: string;
@@ -366,7 +464,11 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
tenantId?: string | null;
}) => Promise<IMongoFile | null>;
deleteFile: (file_id: string) => Promise<IMongoFile | null>;
- deleteFiles: (file_ids: string[], user?: string) => Promise<{ deletedCount?: number }>;
+ deleteFiles: (
+ file_ids: string[],
+ user?: string,
+ options?: DeleteFilesOptions,
+ ) => Promise<DeleteFilesResult>;
deleteFileByFilter: (filter: FilterQuery<IMongoFile>) => Promise<IMongoFile | null>;
batchUpdateFiles: (
updates: Array<{
@@ -570,7 +672,7 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
const query = File.find({
file_id: { $in: fileIds },
user: userId,
- embedded: true,
+ $or: [{ embedded: true }, { 'metadata.searchDestination': true }],
context: FileContext.message_attachment,
// Mongo's null predicate matches explicit null and legacy missing tenant fields.
tenantId: tenantId != null && tenantId !== '' ? tenantId : null,
@@ -588,6 +690,11 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
bytes: 1,
usage: 1,
embedded: 1,
+ 'metadata.searchDestination': 1,
+ 'metadata.routingMimeType': 1,
+ source: 1,
+ storageKey: 1,
+ storageRegion: 1,
context: 1,
expiredAt: 1,
user: 1,
@@ -620,9 +727,9 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
{
user: userId,
tenantId: tenantId != null && tenantId !== '' ? tenantId : null,
- embedded: true,
+ $or: [{ embedded: true }, { 'metadata.searchDestination': true }],
context: FileContext.message_attachment,
- $or: [{ expiredAt: null }, { expiredAt: { $gt: now } }],
+ $and: [{ $or: [{ expiredAt: null }, { expiredAt: { $gt: now } }] }],
},
];
if (excludedFileIds.length > 0) {
@@ -651,6 +758,8 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
bytes: 1,
usage: 1,
embedded: 1,
+ 'metadata.searchDestination': 1,
+ 'metadata.routingMimeType': 1,
context: 1,
user: 1,
tenantId: 1,
@@ -857,6 +966,7 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
$or: [
{ 'metadata.codeEnvRef': { $exists: true } },
{ 'metadata.codeEnvRefs': { $exists: true } },
+ { 'metadata.codeDestination': true },
],
},
ownerScope,
@@ -1027,6 +1137,7 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
$or: [
{ 'metadata.codeEnvRef': { $exists: true } },
{ 'metadata.codeEnvRefs': { $exists: true } },
+ { 'metadata.codeDestination': true },
],
},
ownerScope,
@@ -1069,20 +1180,26 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
? { metadata: { sourceDispatchedAt: data.sourceDispatchedAt } }
: {}),
};
- const result = await File.findOneAndUpdate(
- {
- filename: data.filename,
- conversationId: data.conversationId,
- context: FileContext.execute_code,
- ...tenantFilter,
- },
- { $setOnInsert: insertData },
- /** `timestamps: false`: a claim is an id reservation, not a content
- * write — bumping `updatedAt` here would make the row look freshly
- * written to the background harvest's out-of-order guard, which
- * compares `updatedAt` against the harvest's start time. */
- { upsert: true, new: true, timestamps: false },
- ).lean<IMongoFile>();
+ let result: IMongoFile | null;
+ try {
+ result = await File.findOneAndUpdate(
+ {
+ filename: data.filename,
+ conversationId: data.conversationId,
+ context: FileContext.execute_code,
+ ...tenantFilter,
+ ...codeFileStorageFilter(),
+ },
+ { $setOnInsert: insertData },
+ /** `timestamps: false`: a claim is an id reservation, not a content
+ * write — bumping `updatedAt` here would make the row look freshly
+ * written to the background harvest's out-of-order guard, which
+ * compares `updatedAt` against the harvest's start time. */
+ { upsert: true, new: true, timestamps: false },
+ ).lean<IMongoFile>();
+ } catch (error) {
+ rethrowCodeFileStorageConflict(error);
+ }
if (!result) {
throw new Error(
`[claimCodeFile] Failed to claim file "${data.filename}" for conversation ${data.conversationId}`,
@@ -1111,10 +1228,26 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
delete fileData.expiresAt;
}
- return File.findOneAndUpdate({ file_id: data.file_id }, fileData, {
- new: true,
- upsert: true,
- }).lean<IMongoFile>();
+ const generatedOutput = data.context === FileContext.execute_code;
+ try {
+ return await File.findOneAndUpdate(
+ {
+ file_id: data.file_id,
+ ...(generatedOutput
+ ? {
+ ...codeFileStorageFilter(),
+ user: data.user,
+ tenantId: data.tenantId ?? null,
+ }
+ : {}),
+ },
+ fileData,
+ { new: true, upsert: true },
+ ).lean<IMongoFile>();
+ } catch (error) {
+ if (generatedOutput) rethrowCodeFileStorageConflict(error);
+ throw error;
+ }
}
/**
@@ -1167,6 +1300,9 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
return true;
}
const committed = await updateFile(data, {
+ ...codeFileStorageFilter(),
+ user: data.user,
+ tenantId: data.tenantId ?? null,
$or: [
{ 'metadata.sourceDispatchedAt': { $exists: false } },
{ 'metadata.sourceDispatchedAt': { $lte: sourceDispatchedAt } },
@@ -1209,6 +1345,123 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
).lean<IMongoFile>();
}
+ /** Confirms targeted routes are absent after reconciling deleted snapshots. */
+ async function clearFileCodeEnvRefs(data: ClearFileCodeEnvRefsParams): Promise<boolean> {
+ const File = mongoose.models.File as Model<IMongoFile>;
+ /* Tenant middleware adds its own predicate; it must not replace the snapshot scope. */
+ const query: FilterQuery<IMongoFile> = {
+ file_id: data.file_id,
+ user: data.userId,
+ $and: [{ tenantId: data.tenantId ?? null }],
+ };
+ /* Compare contract fields individually: BSON object equality depends on field order. */
+ const matchesRef = (path: string, ref: CodeEnvRef): FilterQuery<IMongoFile> => {
+ const fields = {
+ kind: ref.kind,
+ id: ref.id,
+ storage_session_id: ref.storage_session_id,
+ file_id: ref.file_id,
+ executionProfile: ref.executionProfile,
+ executionRouteKey: ref.executionRouteKey,
+ provisionedAt: ref.provisionedAt,
+ sandboxFilename: ref.sandboxFilename,
+ version: 'version' in ref ? ref.version : undefined,
+ };
+ const match: FilterQuery<IMongoFile> = {};
+ for (const [field, value] of Object.entries(fields)) {
+ match[`${path}.${field}`] =
+ value === undefined ? { $exists: false } : { $eq: value, $exists: true };
+ }
+ return match;
+ };
+ const matchesByPath = new Map<string, FilterQuery<IMongoFile>[]>();
+ const addMatch = (path: string, ref: CodeEnvRef): void => {
+ const matches = matchesByPath.get(path) ?? [];
+ matches.push(matchesRef(path, ref));
+ matchesByPath.set(path, matches);
+ };
+ for (const { routeKey, ref } of data.refs) {
+ assertValidCodeEnvironmentRouteKey(routeKey);
+ addMatch(`metadata.codeEnvRefs.${routeKey}`, ref);
+ if ((ref.executionRouteKey ?? ref.executionProfile ?? 'default') === routeKey) {
+ addMatch('metadata.codeEnvRef', ref);
+ }
+ }
+ const results = await Promise.allSettled(
+ [...matchesByPath].map(async ([path, matches]) =>
+ File.updateOne(
+ { ...query, $or: matches },
+ { $unset: { [path]: '' }, $set: { 'metadata.codeDestination': true } },
+ { timestamps: false },
+ ),
+ ),
+ );
+ for (const result of results) {
+ if (result.status === 'rejected') {
+ throw result.reason;
+ }
+ }
+ const stored = await File.findOne(query, {
+ _id: 0,
+ 'metadata.codeEnvRef': 1,
+ 'metadata.codeEnvRefs': 1,
+ }).lean<Pick<IMongoFile, 'metadata'>>();
+ if (!stored) {
+ return false;
+ }
+ const remaining = getCodeEnvRefs(stored.metadata);
+ if (data.refs.length === 0) {
+ return remaining.length === 0;
+ }
+ const routes = new Set(data.refs.map(({ routeKey }) => routeKey));
+ return !remaining.some(([routeKey]) => routes.has(routeKey));
+ }
+
+ /** Clears only the embedding snapshot whose vector deletion was confirmed. */
+ async function clearFileEmbeddings(data: ClearFileEmbeddingsParams): Promise<boolean> {
+ const File = mongoose.models.File as Model<IMongoFile>;
+ const query: FilterQuery<IMongoFile> = {
+ file_id: data.file_id,
+ user: data.userId,
+ $and: [{ tenantId: data.tenantId ?? null }],
+ updatedAt: data.updatedAt ?? { $exists: false },
+ $or: [
+ { embedded: true, 'metadata.embeddedEntities': data.embeddedEntities ?? null },
+ { embedded: false, 'metadata.embeddedEntities': { $exists: false } },
+ ],
+ };
+ const result = await File.updateOne(
+ query,
+ {
+ $set: { embedded: false, 'metadata.searchDestination': true },
+ $unset: { 'metadata.embeddedEntities': '' },
+ },
+ { timestamps: false },
+ );
+ return result.matchedCount === 1;
+ }
+
+ /** Checks the final content/presence snapshot before removing primary storage. */
+ async function isFileStorageClean(data: FileStorageCleanupParams): Promise<boolean> {
+ const File = mongoose.models.File as Model<IMongoFile>;
+ const filter = fileStorageCleanupFilter(data);
+ delete filter['storageOperations.deleting'];
+ if ((await File.exists(filter)) != null) return true;
+ if (
+ !data.allowMissingFileRecord ||
+ (data.source !== FileSources.openai && data.source !== FileSources.azure)
+ ) {
+ return false;
+ }
+ return (
+ (await File.exists({
+ file_id: data.file_id,
+ user: data.userId,
+ $and: [{ tenantId: data.tenantId ?? null }],
+ })) == null
+ );
+ }
+
/**
* Records that a file has been embedded into one vector namespace, without disturbing
* the namespaces already recorded. Agents that share a file record, as a duplicate does
@@ -1314,22 +1567,32 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
* keeps a dangling reference, bumping the revision of every project it changes.
*/
async function detachFilesFromProjects(
- files: Pick<IMongoFile, 'file_id' | 'user'>[],
+ files: { file_id: string; user: { toString(): string }; tenantId?: string | null }[],
): Promise<void> {
const ChatProject = mongoose.models.ChatProject as Model<IChatProjectDocument> | undefined;
if (!ChatProject || files.length === 0) {
return;
}
- const idsByUser = new Map<string, string[]>();
+ const idsByScope = new Map<
+ string,
+ { user: string; tenantId: string | null; fileIds: string[] }
+ >();
for (const file of files) {
- const user = String(file.user);
- idsByUser.set(user, [...(idsByUser.get(user) ?? []), file.file_id]);
+ const user = file.user.toString();
+ const tenantId = file.tenantId ?? null;
+ const key = JSON.stringify([user, tenantId]);
+ const scope = idsByScope.get(key);
+ if (scope) {
+ scope.fileIds.push(file.file_id);
+ } else {
+ idsByScope.set(key, { user, tenantId, fileIds: [file.file_id] });
+ }
}
try {
await Promise.all(
- [...idsByUser].map(([user, fileIds]) =>
+ [...idsByScope.values()].map(({ user, tenantId, fileIds }) =>
ChatProject.updateMany(
- { user, file_ids: { $in: fileIds } },
+ { user, $and: [{ tenantId }], file_ids: { $in: fileIds } },
{ $pull: { file_ids: { $in: fileIds } }, $inc: { contextRevision: 1 } },
),
),
@@ -1392,8 +1655,58 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
async function deleteFiles(
file_ids: string[],
user?: string,
- ): Promise<{ deletedCount?: number }> {
+ options?: DeleteFilesOptions,
+ ): Promise<DeleteFilesResult> {
const File = mongoose.models.File as Model<IMongoFile>;
+ if (options) {
+ const ids = new Set(file_ids);
+ if (ids.size === 0) return { deletedCount: 0, deletedFileIds: [], failedFileIds: [] };
+ const snapshots = new Map(options.files.map((file) => [file.file_id, file]));
+ const scopes = [...ids].map((id) => {
+ const file = snapshots.get(id);
+ if (!file) throw new Error('Missing file deletion snapshot');
+ return {
+ file_id: id,
+ userId: file.user?.toString() ?? options.userId,
+ tenantId: file.tenantId,
+ updatedAt: file.updatedAt,
+ source: file.source,
+ embedded: file.embedded,
+ embeddedEntities: file.metadata?.embeddedEntities,
+ };
+ });
+ const deleteQuery = { $or: scopes.map(fileStorageCleanupFilter) };
+ const candidates = await File.find(deleteQuery)
+ .select({ _id: 1, file_id: 1, user: 1, tenantId: 1 })
+ .lean<Pick<IMongoFile, '_id' | 'file_id' | 'user' | 'tenantId'>[]>();
+ const result = await File.deleteMany({
+ ...deleteQuery,
+ _id: { $in: candidates.map((file) => file._id) },
+ });
+ const remaining = await File.find({
+ $or: scopes.map((scope) => ({
+ file_id: scope.file_id,
+ user: scope.userId,
+ $and: [{ tenantId: scope.tenantId ?? null }],
+ })),
+ })
+ .select({ file_id: 1 })
+ .lean<Pick<IMongoFile, 'file_id'>[]>();
+ const remainingIds = new Set(remaining.map((file) => file.file_id));
+ const deletedFileIds: string[] = [];
+ const failedFileIds: string[] = [];
+ const detached = [];
+ for (const scope of scopes) {
+ if (remainingIds.has(scope.file_id)) {
+ failedFileIds.push(scope.file_id);
+ } else {
+ deletedFileIds.push(scope.file_id);
+ detached.push({ file_id: scope.file_id, user: scope.userId, tenantId: scope.tenantId });
+ }
+ }
+ await detachFilesFromProjects(detached);
+ return { deletedCount: result.deletedCount, deletedFileIds, failedFileIds };
+ }
if (user) {
const result = await File.deleteMany({ user });
await detachAllUserFilesFromProjects(user);
@@ -1401,7 +1714,7 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
}
const deleteQuery: FilterQuery<IMongoFile> = { file_id: { $in: file_ids } };
const doomed = await File.find(deleteQuery)
- .select({ file_id: 1, user: 1 })
+ .select({ file_id: 1, user: 1, tenantId: 1 })
.lean<IMongoFile[]>();
const result = await File.deleteMany(deleteQuery);
await detachFilesFromProjects(doomed);
@@ -1627,7 +1940,10 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
createFile,
updateFile,
updateFileCodeEnvRef,
+ clearFileCodeEnvRefs,
addFileEmbeddedEntity,
+ clearFileEmbeddings,
+ isFileStorageClean,
adoptLegacySearchEmbeddings,
updateFileUsage,
deleteFile,
diff --git a/packages/data-schemas/src/methods/fileStorage.spec.ts b/packages/data-schemas/src/methods/fileStorage.spec.ts
new file mode 100644
index 0000000000..0b1142b87b
--- /dev/null
+++ b/packages/data-schemas/src/methods/fileStorage.spec.ts
@@ -0,0 +1,409 @@
+import mongoose from 'mongoose';
+import { MongoMemoryServer } from 'mongodb-memory-server';
+import type { FileStorageClaim, IMongoFile } from '~/types/file';
+import type { AllMethods, FileStorageMethods } from './index';
+import { _resetStrictCache } from '~/models/plugins/tenantIsolation';
+import { tenantStorage, runAsSystem } from '~/config/tenantContext';
+import { createFileStorageMethods } from './fileStorage';
+import { createModels } from '~/models';
+
+let File: mongoose.Model<IMongoFile>;
+let methods: FileStorageMethods;
+let peer: FileStorageMethods;
+let mongoServer: MongoMemoryServer;
+let modelNames: string[];
+const user = new mongoose.Types.ObjectId();
+const updatedAt = new Date('2026-10-20T00:00:00Z');
+const claim: FileStorageClaim = {
+ file_id: 'storage-fence',
+ userId: user.toString(),
+ tenantId: 'tenant-a',
+ token: 'owner-token',
+};
+const deletion = { ...claim, updatedAt };
+const originalStrict = process.env.TENANT_ISOLATION_STRICT;
+
+const seed = (overrides = {}) =>
+ File.collection.insertOne({
+ file_id: claim.file_id,
+ user,
+ tenantId: claim.tenantId,
+ filename: 'notes.txt',
+ filepath: '/uploads/notes.txt',
+ type: 'text/plain',
+ bytes: 100,
+ source: 'local',
+ createdAt: updatedAt,
+ updatedAt,
+ metadata: { futureField: 'preserve-me' },
+ ...overrides,
+ });
+const read = () => File.collection.findOne({ file_id: claim.file_id, user });
+
+beforeAll(async () => {
+ delete process.env.TENANT_ISOLATION_STRICT;
+ _resetStrictCache();
+ mongoServer = await MongoMemoryServer.create({ instance: { args: ['--nounixsocket'] } });
+ await mongoose.connect(mongoServer.getUri());
+ const models = createModels(mongoose);
+ modelNames = Object.keys(models);
+ Object.assign(mongoose.models, models);
+ File = mongoose.models.File as mongoose.Model<IMongoFile>;
+ methods = createFileStorageMethods(mongoose);
+ peer = createFileStorageMethods(mongoose);
+}, 30_000);
+
+beforeEach(async () => {
+ await runAsSystem(() => File.deleteMany({}));
+});
+
+afterAll(async () => {
+ await mongoose.disconnect();
+ if (mongoServer) await mongoServer.stop();
+ for (const name of modelNames ?? []) delete mongoose.models[name];
+ if (originalStrict === undefined) delete process.env.TENANT_ISOLATION_STRICT;
+ else process.env.TENANT_ISOLATION_STRICT = originalStrict;
+ _resetStrictCache();
+});
+
+describe('file storage fence', () => {
+ it('exports every storage claim method through the shared factory contract', () => {
+ const publicMethods: Pick<AllMethods, keyof FileStorageMethods> = methods;
+ expect(Object.keys(publicMethods)).toHaveLength(5);
+ });
+ it('retains every concurrent provisioning owner until release', async () => {
+ await seed();
+ const claims = ['code-a', 'code-b', 'search'].map((token) => ({ ...claim, token }));
+ expect(await Promise.all(claims.map(methods.acquireFileProvision))).toEqual([true, true, true]);
+ expect(await peer.acquireFileProvision(claims[0])).toBe(true);
+ expect((await read())?.storageOperations?.provisioning?.sort()).toEqual([
+ 'code-a',
+ 'code-b',
+ 'search',
+ ]);
+ expect(await peer.acquireFileDeletion(deletion)).toBeNull();
+ expect(await peer.releaseFileProvision({ ...claim, token: 'thief' })).toBe(true);
+ expect((await read())?.storageOperations?.provisioning).toContain(claims[0].token);
+ expect(await methods.releaseFileProvision(claims[0])).toBe(true);
+ expect(await peer.acquireFileDeletion(deletion)).toBeNull();
+ expect(await Promise.all(claims.slice(1).map(methods.releaseFileProvision))).toEqual([
+ true,
+ true,
+ ]);
+ expect(await peer.acquireFileDeletion(deletion)).toEqual({ resumed: false });
+ });
+
+ it('blocks provisioning when deletion wins first, without automatic takeover', async () => {
+ await seed();
+ expect(await methods.acquireFileDeletion(deletion)).toEqual({ resumed: false });
+ expect(await peer.acquireFileProvision({ ...claim, token: 'late' })).toBe(false);
+ expect(await peer.acquireFileDeletion({ ...deletion, token: 'other-delete' })).toBeNull();
+ expect(await methods.acquireFileDeletion(deletion)).toBeNull();
+ expect(
+ await peer.releaseFileDeletion({ ...claim, token: 'thief', retainDeletion: false }),
+ ).toBe(false);
+ expect((await read())?.storageOperations).toEqual({
+ deleting: claim.token,
+ deletionPending: true,
+ });
+ });
+
+ it('atomically chooses exactly one winner in a cross-factory provision/delete race', async () => {
+ await seed();
+ const [provisioned, deleted] = await Promise.all([
+ methods.acquireFileProvision(claim),
+ peer.acquireFileDeletion(deletion),
+ ]);
+ expect(Number(provisioned) + Number(deleted !== null)).toBe(1);
+ const operations = (await read())?.storageOperations;
+ if (provisioned) {
+ expect(operations).toEqual({ provisioning: [claim.token] });
+ } else {
+ expect(operations).toEqual({ deleting: claim.token, deletionPending: true });
+ }
+ });
+
+ it('admits only one of simultaneous delete claimants', async () => {
+ await seed();
+ const results = await Promise.all([
+ methods.acquireFileDeletion(deletion),
+ peer.acquireFileDeletion({ ...deletion, token: 'peer' }),
+ ]);
+ expect(results.filter((result) => result !== null)).toEqual([{ resumed: false }]);
+ });
+
+ it('clears pending on pre-primary failure to permit lazy repair', async () => {
+ await seed();
+ expect(await methods.acquireFileDeletion(deletion)).toEqual({ resumed: false });
+ expect(await methods.releaseFileDeletion({ ...claim, retainDeletion: false })).toBe(true);
+ expect((await read())?.storageOperations).toEqual({});
+ expect(await peer.acquireFileProvision(claim)).toBe(true);
+ expect(await peer.releaseFileProvision(claim)).toBe(true);
+ expect(await peer.acquireFileDeletion(deletion)).toEqual({ resumed: false });
+ });
+
+ it('retains primary-success intent and resumes without reprovisioning', async () => {
+ await seed();
+ expect(await methods.acquireFileDeletion(deletion)).toEqual({ resumed: false });
+ expect(await methods.releaseFileDeletion({ ...claim, retainDeletion: true })).toBe(true);
+ expect((await read())?.storageOperations).toEqual({ deletionPending: true });
+ expect(await peer.acquireFileProvision(claim)).toBe(false);
+ const retry = { ...deletion, token: 'retry' };
+ expect(await peer.acquireFileDeletion(retry)).toEqual({ resumed: true });
+ expect(await methods.releaseFileDeletion({ ...claim, retainDeletion: false })).toBe(false);
+ expect(await peer.releaseFileDeletion({ ...retry, retainDeletion: true })).toBe(true);
+ expect(await methods.acquireFileDeletion(deletion)).toEqual({ resumed: true });
+ });
+
+ it('requires the deletion content snapshot but not a provisioning snapshot', async () => {
+ await seed();
+ expect(await methods.acquireFileDeletion({ ...deletion, updatedAt: new Date(0) })).toBeNull();
+ expect(await methods.acquireFileDeletion(claim)).toBeNull();
+ expect(await methods.acquireFileProvision(claim)).toBe(true);
+ expect(await methods.releaseFileProvision(claim)).toBe(true);
+ expect(
+ await methods.acquireFileDeletion({ ...deletion, updatedAt: updatedAt.toISOString() }),
+ ).toEqual({ resumed: false });
+ });
+
+ it('can claim an explicitly absent legacy snapshot', async () => {
+ await seed();
+ await File.collection.updateOne({ file_id: claim.file_id }, { $unset: { updatedAt: '' } });
+ expect(await methods.acquireFileDeletion(claim)).toEqual({ resumed: false });
+ });
+
+ it('persists a code result only for its active owner and tenant claim', async () => {
+ await seed();
+ const code = {
+ routeKey: 'default',
+ ref: {
+ kind: 'user' as const,
+ id: claim.userId,
+ storage_session_id: 'session',
+ file_id: 'remote',
+ },
+ };
+ expect(await methods.commitFileProvision({ claim, code })).toBe(false);
+ expect(await methods.acquireFileProvision(claim)).toBe(true);
+ expect(await methods.commitFileProvision({ claim: { ...claim, token: 'stale' }, code })).toBe(
+ false,
+ );
+ expect(await methods.commitFileProvision({ claim, code })).toBe(true);
+ expect((await read())?.metadata?.codeEnvRefs?.default).toEqual(code.ref);
+ expect(
+ await peer.acquireFileDeletion({ ...deletion, updatedAt: (await read())?.updatedAt }),
+ ).toBeNull();
+ expect(await methods.releaseFileProvision(claim)).toBe(true);
+ expect(await methods.commitFileProvision({ claim, code })).toBe(false);
+ });
+
+ it('records a vector namespace without releasing the provisioning claim early', async () => {
+ await seed();
+ expect(await methods.acquireFileProvision(claim)).toBe(true);
+ expect(
+ await methods.commitFileProvision({ claim, embedded: true, entityId: 'namespace' }),
+ ).toBe(true);
+ const stored = await read();
+ expect(stored?.embedded).toBe(true);
+ expect(stored?.metadata?.embeddedEntities).toEqual(['namespace']);
+ expect(stored?.storageOperations?.provisioning).toEqual([claim.token]);
+ });
+
+ it('cannot complete a stale upload against a recreated file identity', async () => {
+ await seed();
+ expect(await methods.acquireFileProvision(claim)).toBe(true);
+ await File.collection.deleteMany({});
+ await seed();
+ expect(await methods.commitFileProvision({ claim, embedded: true })).toBe(false);
+ expect(await methods.releaseFileProvision(claim)).toBe(true);
+ expect((await read())?.embedded).toBeUndefined();
+ });
+
+ it.each(['provision', 'delete'])(
+ 'retains the durable retry anchor beyond upload TTL while holding %s',
+ async (operation) => {
+ await seed({ expiresAt: new Date(Date.now() + 60_000) });
+ if (operation === 'provision') expect(await methods.acquireFileProvision(claim)).toBe(true);
+ else expect(await methods.acquireFileDeletion(deletion)).toEqual({ resumed: false });
+ expect((await read())?.expiresAt).toBeUndefined();
+ expect((await read())?.updatedAt).toEqual(updatedAt);
+ },
+ );
+
+ it('never upserts missing files for any claim or release', async () => {
+ expect(await methods.acquireFileProvision(claim)).toBe(false);
+ expect(await methods.releaseFileProvision(claim)).toBe(false);
+ expect(await methods.acquireFileDeletion(deletion)).toBeNull();
+ expect(await methods.releaseFileDeletion({ ...claim, retainDeletion: false })).toBe(false);
+ expect(await File.collection.countDocuments()).toBe(0);
+ });
+
+ it.each([
+ { userId: new mongoose.Types.ObjectId().toString() },
+ { tenantId: 'tenant-b' },
+ { tenantId: undefined },
+ { tenantId: null },
+ { file_id: 'other-file' },
+ ])('isolates every claim and release for scope %j', async (scope) => {
+ await seed();
+ const other = { ...claim, ...scope };
+ expect(await runAsSystem(() => peer.acquireFileProvision(other))).toBe(false);
+ expect(await runAsSystem(() => peer.acquireFileDeletion({ ...other, updatedAt }))).toBeNull();
+ expect(await methods.acquireFileProvision(claim)).toBe(true);
+ expect(await runAsSystem(() => peer.releaseFileProvision(other))).toBe(false);
+ expect(await methods.releaseFileProvision(claim)).toBe(true);
+ expect(await methods.acquireFileDeletion(deletion)).toEqual({ resumed: false });
+ expect(
+ await runAsSystem(() => peer.releaseFileDeletion({ ...other, retainDeletion: false })),
+ ).toBe(false);
+ expect((await read())?.storageOperations?.deleting).toBe(claim.token);
+ });
+
+ it.each([undefined, null])(
+ 'explicitly scopes legacy tenant %j even in system context',
+ async (tenantId) => {
+ await seed({ tenantId: undefined });
+ await seed({ tenantId: 'tenant-a' });
+ const legacy = { ...claim, tenantId };
+ expect(await runAsSystem(() => methods.acquireFileProvision(legacy))).toBe(true);
+ const rows = await File.collection.find({ file_id: claim.file_id }).toArray();
+ expect(rows.find((row) => row.tenantId === 'tenant-a')?.storageOperations).toBeUndefined();
+ expect(await runAsSystem(() => methods.releaseFileProvision(legacy))).toBe(true);
+ expect(
+ await runAsSystem(() => methods.acquireFileDeletion({ ...legacy, updatedAt })),
+ ).toEqual({
+ resumed: false,
+ });
+ expect(
+ await runAsSystem(() => methods.releaseFileDeletion({ ...legacy, retainDeletion: true })),
+ ).toBe(true);
+ },
+ );
+
+ it('cannot have its explicit tenant overwritten by ambient tenant middleware', async () => {
+ await seed();
+ await seed({ tenantId: 'tenant-b' });
+ const context = { tenantId: 'tenant-b' };
+ expect(await tenantStorage.run(context, () => methods.acquireFileProvision(claim))).toBe(false);
+ expect(
+ await tenantStorage.run(context, () => methods.acquireFileDeletion(deletion)),
+ ).toBeNull();
+ expect(await methods.acquireFileProvision(claim)).toBe(true);
+ expect(await tenantStorage.run(context, () => methods.releaseFileProvision(claim))).toBe(false);
+ expect(await methods.releaseFileProvision(claim)).toBe(true);
+ expect(await methods.acquireFileDeletion(deletion)).toEqual({ resumed: false });
+ expect(
+ await tenantStorage.run(context, () =>
+ methods.releaseFileDeletion({ ...claim, retainDeletion: false }),
+ ),
+ ).toBe(false);
+ expect(
+ await File.collection.countDocuments({
+ storageOperations: { $exists: true },
+ tenantId: 'tenant-b',
+ }),
+ ).toBe(0);
+ });
+
+ it('preserves timestamps, content, metadata, and unknown fields', async () => {
+ await seed({
+ futureField: { value: 'untouched' },
+ storageOperations: { futureFence: 'untouched' },
+ });
+ const before = await read();
+ const unchanged = async () => {
+ const after = await read();
+ expect(after).toEqual({
+ ...before,
+ storageOperations: { ...after?.storageOperations, futureFence: 'untouched' },
+ });
+ };
+ expect(await methods.acquireFileProvision(claim)).toBe(true);
+ await unchanged();
+ expect(await methods.releaseFileProvision(claim)).toBe(true);
+ await unchanged();
+ expect(await methods.acquireFileDeletion(deletion)).toEqual({ resumed: false });
+ await unchanged();
+ expect(await methods.releaseFileDeletion({ ...claim, retainDeletion: true })).toBe(true);
+ await unchanged();
+ expect(await methods.acquireFileDeletion(deletion)).toEqual({ resumed: true });
+ expect(await methods.releaseFileDeletion({ ...claim, retainDeletion: false })).toBe(true);
+ await unchanged();
+ });
+
+ it('propagates database failures instead of reporting contention or absence', async () => {
+ const disconnected = new mongoose.Mongoose();
+ disconnected.model<IMongoFile>('File', File.schema.clone().set('bufferCommands', false));
+ const unavailable = createFileStorageMethods(disconnected);
+ await expect(unavailable.acquireFileProvision(claim)).rejects.toThrow();
+ await expect(unavailable.releaseFileProvision(claim)).rejects.toThrow();
+ await expect(unavailable.acquireFileDeletion(deletion)).rejects.toThrow();
+ await expect(
+ unavailable.releaseFileDeletion({ ...claim, retainDeletion: false }),
+ ).rejects.toThrow();
+ });
+
+ it.each([
+ { token: '' },
+ { token: ' ' },
+ { token: null },
+ { token: { $ne: '' } },
+ { userId: '' },
+ { userId: 'not-an-object-id' },
+ { userId: { $ne: null } },
+ { file_id: '' },
+ { tenantId: '' },
+ ])('rejects invalid claim scope %j without writing', async (invalid) => {
+ await seed();
+ const before = await read();
+ // Exercise untrusted runtime input without weakening the public contract.
+ const data = { ...claim, ...invalid } as FileStorageClaim;
+ await expect(methods.acquireFileProvision(data)).rejects.toThrow('valid file storage');
+ await expect(methods.releaseFileProvision(data)).rejects.toThrow('valid file storage');
+ await expect(methods.acquireFileDeletion({ ...data, updatedAt })).rejects.toThrow(
+ 'valid file storage',
+ );
+ await expect(methods.releaseFileDeletion({ ...data, retainDeletion: false })).rejects.toThrow(
+ 'valid file storage',
+ );
+ expect(await read()).toEqual(before);
+ });
+});
+
+describe('deletion release acknowledgement recovery', () => {
+ it.each([true, false])(
+ 'acknowledges repeated release without changing pending intent, retain=%s',
+ async (retainDeletion) => {
+ await seed();
+ expect(await methods.acquireFileDeletion(deletion)).toEqual({ resumed: false });
+ expect(await methods.releaseFileDeletion({ ...claim, retainDeletion })).toBe(true);
+ expect(await peer.releaseFileDeletion({ ...claim, retainDeletion: !retainDeletion })).toBe(
+ true,
+ );
+ expect((await read())?.storageOperations?.deletionPending).toBe(
+ retainDeletion ? true : undefined,
+ );
+ expect((await read())?.storageOperations?.deleting).toBeUndefined();
+ },
+ );
+
+ it('does not acknowledge a release for a missing record, another owner, or another tenant', async () => {
+ await seed();
+ for (const wrong of [
+ { ...claim, file_id: 'missing' },
+ { ...claim, userId: new mongoose.Types.ObjectId().toString() },
+ { ...claim, tenantId: 'other-tenant' },
+ ])
+ expect(await methods.releaseFileDeletion({ ...wrong, retainDeletion: false })).toBe(false);
+ });
+
+ it('never releases a newer deletion token when replaying an old release', async () => {
+ await seed();
+ expect(await methods.acquireFileDeletion(deletion)).not.toBeNull();
+ expect(await methods.releaseFileDeletion({ ...claim, retainDeletion: false })).toBe(true);
+ expect(await peer.acquireFileDeletion({ ...deletion, token: 'new-token' })).not.toBeNull();
+ expect(await methods.releaseFileDeletion({ ...claim, retainDeletion: false })).toBe(false);
+ expect((await read())?.storageOperations?.deleting).toBe('new-token');
+ expect((await read())?.storageOperations?.deletionPending).toBe(true);
+ });
+});
diff --git a/packages/data-schemas/src/methods/fileStorage.ts b/packages/data-schemas/src/methods/fileStorage.ts
new file mode 100644
index 0000000000..cb266de52a
--- /dev/null
+++ b/packages/data-schemas/src/methods/fileStorage.ts
@@ -0,0 +1,181 @@
+import type { FilterQuery, Model } from 'mongoose';
+import type {
+ FileStorageClaim,
+ FileStorageDeleteClaim,
+ FileProvisionCommit,
+ IMongoFile,
+} from '~/types/file';
+
+function claimFilter(data: FileStorageClaim): FilterQuery<IMongoFile> {
+ if (
+ typeof data.file_id !== 'string' ||
+ data.file_id.trim().length === 0 ||
+ typeof data.userId !== 'string' ||
+ !/^[a-f\d]{24}$/i.test(data.userId) ||
+ typeof data.token !== 'string' ||
+ data.token.trim().length === 0 ||
+ (data.tenantId != null &&
+ (typeof data.tenantId !== 'string' || data.tenantId.trim().length === 0))
+ ) {
+ throw new Error('A valid file storage owner scope and claim token are required');
+ }
+ return {
+ file_id: data.file_id,
+ user: data.userId,
+ $and: [{ tenantId: data.tenantId ?? null }],
+ };
+}
+
+/** Claims never expire: elapsed time cannot fence an external storage operation. */
+export function createFileStorageMethods(mongoose: typeof import('mongoose')): {
+ acquireFileProvision: (data: FileStorageClaim) => Promise<boolean>;
+ releaseFileProvision: (data: FileStorageClaim) => Promise<boolean>;
+ commitFileProvision: (data: FileProvisionCommit) => Promise<boolean>;
+ acquireFileDeletion: (data: FileStorageDeleteClaim) => Promise<{ resumed: boolean } | null>;
+ releaseFileDeletion: (data: FileStorageClaim & { retainDeletion: boolean }) => Promise<boolean>;
+} {
+ async function acquireFileProvision(data: FileStorageClaim): Promise<boolean> {
+ const File = mongoose.models.File as Model<IMongoFile>;
+ const result = await File.updateOne(
+ {
+ ...claimFilter(data),
+ 'storageOperations.deletionPending': { $ne: true },
+ 'storageOperations.deleting': { $exists: false },
+ },
+ { $addToSet: { 'storageOperations.provisioning': data.token }, $unset: { expiresAt: '' } },
+ { timestamps: false },
+ );
+ return result.matchedCount === 1;
+ }
+
+ /** An already-applied release is successful; only the supplied token is removed. */
+ async function releaseFileProvision(data: FileStorageClaim): Promise<boolean> {
+ const File = mongoose.models.File as Model<IMongoFile>;
+ const result = await File.updateOne(
+ claimFilter(data),
+ { $pull: { 'storageOperations.provisioning': data.token } },
+ { timestamps: false },
+ );
+ return result.matchedCount === 1;
+ }
+
+ async function commitFileProvision(data: FileProvisionCommit): Promise<boolean> {
+ const File = mongoose.models.File as Model<IMongoFile>;
+ if (data.releaseOnly) {
+ return (
+ (await File.exists({
+ ...claimFilter(data.claim),
+ 'storageOperations.provisioning': data.claim.token,
+ 'storageOperations.deletionPending': { $ne: true },
+ 'storageOperations.deleting': { $exists: false },
+ })) != null
+ );
+ }
+ const update: Record<string, unknown> = {};
+ if (data.output) {
+ const { file } = data.output;
+ if (
+ file.file_id !== data.claim.file_id ||
+ file.user !== data.claim.userId ||
+ (file.tenantId ?? null) !== (data.claim.tenantId ?? null)
+ ) {
+ throw new Error('Generated output does not match its storage claim');
+ }
+ Object.assign(update, file);
+ delete update.file_id;
+ }
+ if (data.code) {
+ const { routeKey, ref, legacyRef } = data.code;
+ if (!routeKey || routeKey.includes('.') || routeKey.startsWith('$')) {
+ throw new Error('Invalid code environment route key');
+ }
+ update[`metadata.codeEnvRefs.${routeKey}`] = ref;
+ if (legacyRef) update['metadata.codeEnvRef'] = legacyRef;
+ }
+ if (data.embedded === true) update.embedded = true;
+ if (Object.keys(update).length === 0) throw new Error('Missing provisioning result');
+ const result = await File.updateOne(
+ {
+ ...claimFilter(data.claim),
+ 'storageOperations.provisioning': data.claim.token,
+ 'storageOperations.deletionPending': { $ne: true },
+ 'storageOperations.deleting': { $exists: false },
+ ...(data.output?.sourceDispatchedAt != null
+ ? {
+ $or: [
+ { 'metadata.sourceDispatchedAt': { $exists: false } },
+ { 'metadata.sourceDispatchedAt': { $lte: data.output.sourceDispatchedAt } },
+ ],
+ }
+ : {}),
+ },
+ {
+ $set: update,
+ ...(data.entityId ? { $addToSet: { 'metadata.embeddedEntities': data.entityId } } : {}),
+ $unset: { expiresAt: '' },
+ },
+ );
+ return result.matchedCount === 1;
+ }
+
+ async function acquireFileDeletion(
+ data: FileStorageDeleteClaim,
+ ): Promise<{ resumed: boolean } | null> {
+ const File = mongoose.models.File as Model<IMongoFile>;
+ const previous = await File.findOneAndUpdate(
+ {
+ ...claimFilter(data),
+ updatedAt: data.updatedAt ?? { $exists: false },
+ 'storageOperations.deleting': { $exists: false },
+ 'storageOperations.provisioning.0': { $exists: false },
+ },
+ {
+ $set: {
+ 'storageOperations.deleting': data.token,
+ 'storageOperations.deletionPending': true,
+ },
+ $unset: { expiresAt: '' },
+ },
+ { new: false, timestamps: false },
+ )
+ .select({ 'storageOperations.deletionPending': 1 })
+ .lean();
+ return previous == null
+ ? null
+ : { resumed: previous.storageOperations?.deletionPending === true };
+ }
+
+ async function releaseFileDeletion(
+ data: FileStorageClaim & { retainDeletion: boolean },
+ ): Promise<boolean> {
+ const File = mongoose.models.File as Model<IMongoFile>;
+ const result = await File.updateOne(
+ { ...claimFilter(data), 'storageOperations.deleting': data.token },
+ {
+ $unset: {
+ 'storageOperations.deleting': '',
+ ...(!data.retainDeletion ? { 'storageOperations.deletionPending': '' } : {}),
+ },
+ },
+ { timestamps: false },
+ );
+ if (result.matchedCount === 1) return true;
+ // A lost release acknowledgement must not clear pending intent or a newer token.
+ return (
+ (await File.exists({
+ ...claimFilter(data),
+ 'storageOperations.deleting': { $exists: false },
+ })) != null
+ );
+ }
+
+ return {
+ acquireFileProvision,
+ releaseFileProvision,
+ commitFileProvision,
+ acquireFileDeletion,
+ releaseFileDeletion,
+ };
+}
+
+export type FileStorageMethods = ReturnType<typeof createFileStorageMethods>;
diff --git a/packages/data-schemas/src/methods/index.ts b/packages/data-schemas/src/methods/index.ts
index 64be35bad8..4c907cc5b6 100644
--- a/packages/data-schemas/src/methods/index.ts
+++ b/packages/data-schemas/src/methods/index.ts
@@ -8,6 +8,8 @@ export type {
} from './subagentContent';
import type { ToolApprovalGrantStorage } from 'librechat-data-provider';
import type { ScheduleMCPConsentStorage } from './scheduleConsent';
+import type { ClearFileCodeEnvRefsParams } from '~/types/file';
+import type { FileStorageMethods } from './fileStorage';
import { createToolApprovalGrantMethods } from './toolApprovalGrant';
import { createScheduleMCPConsentStorage } from './scheduleConsent';
export { createScheduleMCPConsentStorage } from './scheduleConsent';
@@ -15,6 +17,7 @@ export type { ScheduleMCPConsentStorage, ScheduleConsentSnapshot } from './sched
import type {
FileMethods,
FileOwnerScope,
+ ClearFileEmbeddingsParams,
AvailableProjectFileRecord,
AvailableProjectFilesOptions,
AvailableProjectFilesResult,
@@ -42,7 +45,9 @@ import { createTokenMethods, type TokenMethods } from './token';
import { createTokenListMethods, type TokenListMethods } from './tokenList';
import { createRoleMethods, RoleConflictError } from './role';
import { createKeyMethods, type KeyMethods } from './key';
+import { createFileStorageMethods } from './fileStorage';
import { createFileMethods } from './file';
+export { createFileStorageMethods } from './fileStorage';
/* Memories */
import { createMemoryMethods, type MemoryMethods } from './memory';
/* Tool Favorites */
@@ -281,6 +286,7 @@ export type AllMethods = SubagentContentMethods &
RoleMethods &
KeyMethods &
FileMethods &
+ FileStorageMethods &
MemoryMethods &
ToolFavoriteMethods &
AgentCategoryMethods &
@@ -524,6 +530,7 @@ export function createMethods(
...roleMethods,
...createKeyMethods(mongoose),
...createFileMethods(mongoose),
+ ...createFileStorageMethods(mongoose),
...createMemoryMethods(mongoose),
...createToolFavoriteMethods(mongoose),
...createAgentCategoryMethods(mongoose),
@@ -595,8 +602,11 @@ export type {
KeyMethods,
MemoryMethods,
FileMethods,
+ FileStorageMethods,
FileOwnerScope,
AvailableProjectFileRecord,
+ ClearFileCodeEnvRefsParams,
+ ClearFileEmbeddingsParams,
AvailableProjectFilesOptions,
AvailableProjectFilesResult,
ProjectFileRecord,
diff --git a/packages/data-schemas/src/schema/file.ts b/packages/data-schemas/src/schema/file.ts
index f8445ad9dc..3e2fc7a644 100644
--- a/packages/data-schemas/src/schema/file.ts
+++ b/packages/data-schemas/src/schema/file.ts
@@ -62,6 +62,11 @@ const file: Schema<IMongoFile> = new Schema(
storageRegion: {
type: String,
},
+ storageOperations: {
+ provisioning: { type: [String], default: undefined },
+ deleting: { type: String, default: undefined },
+ deletionPending: { type: Boolean, default: undefined },
+ },
object: {
type: String,
required: true,
@@ -163,6 +168,16 @@ const file: Schema<IMongoFile> = new Schema(
type: [String],
default: undefined,
},
+ /** Retains the code destination when confirmed deletion clears its pointers. */
+ codeDestination: {
+ type: Boolean,
+ default: undefined,
+ },
+ /** Retains the search destination when confirmed deletion clears its vectors. */
+ searchDestination: {
+ type: Boolean,
+ default: undefined,
+ },
/** The user named this file's destination, through the chooser or by requesting a
* tool resource, so the destinations absent from the record were declined rather
* than deferred. Unlike the endpoint setting, which the next turn may resolve
@@ -224,6 +239,13 @@ file.index(
{ user: 1, tenantId: 1, context: 1, _id: -1 },
{ name: 'project_file_picker', partialFilterExpression: { embedded: true } },
);
+file.index(
+ { user: 1, tenantId: 1, context: 1, _id: -1 },
+ {
+ name: 'project_file_picker_search_destination',
+ partialFilterExpression: { 'metadata.searchDestination': true },
+ },
+);
file.index({ createdAt: 1, updatedAt: 1 });
file.index(
{ filename: 1, conversationId: 1, context: 1, tenantId: 1 },
diff --git a/packages/data-schemas/src/types/file.ts b/packages/data-schemas/src/types/file.ts
index 012f9bbb07..4b9d243f74 100644
--- a/packages/data-schemas/src/types/file.ts
+++ b/packages/data-schemas/src/types/file.ts
@@ -7,6 +7,65 @@ import type {
TFile,
} from 'librechat-data-provider';
+export type FileStorageScope = {
+ file_id: string;
+ userId: string;
+ tenantId?: string | null;
+};
+
+export type FileStorageClaim = FileStorageScope & {
+ token: string;
+};
+
+export type FileProvisionCommit = {
+ claim: FileStorageClaim;
+ code?: { routeKey: string; ref: CodeEnvRef; legacyRef?: CodeEnvRef };
+ entityId?: string;
+ embedded?: boolean;
+ output?: { file: CodeFileCommitData; sourceDispatchedAt?: number };
+ releaseOnly?: boolean;
+};
+
+export type FileStorageDeleteClaim = FileStorageClaim & {
+ updatedAt?: Date | string;
+};
+
+/** Content snapshot whose primary and secondary storage cleanup must agree. */
+export type FileStorageCleanupParams = {
+ file_id: string;
+ userId: string;
+ tenantId?: string | null;
+ updatedAt?: Date | string;
+ source?: string;
+ embedded?: boolean;
+ embeddedEntities?: string[];
+ /** Authorized provider-only assistant files have no persisted local snapshot. */
+ allowMissingFileRecord?: boolean;
+};
+
+export type FileDeletionSnapshot = Omit<FileStorageCleanupParams, 'userId' | 'embeddedEntities'> & {
+ user?: { toString(): string };
+ metadata?: { embeddedEntities?: string[] };
+};
+
+export type DeleteFilesResult = {
+ deletedCount?: number;
+ deletedFileIds?: string[];
+ failedFileIds?: string[];
+};
+
+export type DeleteFilesOptions = {
+ files: FileDeletionSnapshot[];
+ userId: string;
+};
+
+export type ClearFileCodeEnvRefsParams = {
+ file_id: string;
+ userId: string;
+ tenantId?: string;
+ refs: Array<{ routeKey: string; ref: CodeEnvRef }>;
+};
+
export type RunArtifactRunScope = {
userId: string;
tenantId?: string | null;
@@ -134,6 +193,11 @@ export interface IMongoFile extends Omit<Document, 'model'> {
filepath: string;
storageKey?: string;
storageRegion?: string;
+ storageOperations?: {
+ provisioning?: string[];
+ deleting?: string;
+ deletionPending?: boolean;
+ };
object: 'file';
embedded?: boolean;
type: string;
@@ -157,6 +221,10 @@ export interface IMongoFile extends Omit<Document, 'model'> {
sourceDispatchedAt?: number;
/** Vector namespaces this file has been embedded into. */
embeddedEntities?: string[];
+ /** Code remains a destination even after its remote pointers are cleared. */
+ codeDestination?: boolean;
+ /** Search remains a destination even after its vectors are cleared. */
+ searchDestination?: boolean;
/** The user named this destination, so absent ones were declined. */
destinationChosen?: boolean;
/** The type the delivery route was resolved against, when conversion changed it. */