Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
import { Types } from 'mongoose';
import { createHash } from 'crypto';
import { EToolResources, FileContext, FileSources, getCodeEnvRefs } from 'librechat-data-provider';
import type {
TFile,
CodeEnvRef,
TFileWorkspaceImport,
TFileWorkspaceImports,
} from 'librechat-data-provider';
import type { FilterQuery, SortOrder, Model } from 'mongoose';
import type {
IMongoFile,
RunArtifactFile,
RunArtifactClaim,
RunArtifactScope,
CodeFileCommitData,
RunArtifactRunScope,
PublishRunArtifactInput,
ClearFileCodeEnvRefsParams,
FileStorageCleanupParams,
DeleteFilesOptions,
DeleteFilesResult,
} from '~/types/file';
import type { IChatProjectDocument } from '~/types';
import { tenantSafeBulkWrite } from '~/utils/tenantBulkWrite';
import { fileContentUpdate } from './fileContent';
import { escapeRegExp } from '~/utils/string';
import logger from '../config/winston';
/** A loaded file record, named by the fields that identify it: `file_id` is not unique alone. */
export type LegacySearchFileRef = {
file_id: string;
user: string;
tenantId?: string;
};
/** A user's file's freshly computed digest, the copy placed in one workspace, or both. */
export type WorkspaceImportRecord = {
file_id: string;
user: string;
/** The record's own tenant; an untenanted record is matched as untenanted. */
tenantId?: string | null;
/** `key` is an opaque hex digest of the conversation and workspace the copy went to. */
copy?: { key: string; target: TFileWorkspaceImport };
digest?: NonNullable<TFileWorkspaceImports['digest']>;
};
export type ClearFileEmbeddingsParams = {
file_id: string;
userId: string;
tenantId?: string | null;
updatedAt?: Date | string;
embeddedEntities?: string[];
ownershipTrackingStart?: string;
};
export const DEFAULT_AVAILABLE_PROJECT_FILES_LIMIT = 20;
export const MAX_AVAILABLE_PROJECT_FILES_LIMIT = 50;
/** Plain projection of a candidate file. The picker's response crosses into `@librechat/api`
* and the client, so this names no Mongoose type: `_id` and `user` are already strings here. */
export type AvailableProjectFileRecord = {
_id: string;
file_id: string;
filename: string;
filepath: string;
object: 'file';
type: string;
bytes: number;
usage: number;
embedded?: boolean;
metadata?: { searchDestination?: boolean; routingMimeType?: string };
context?: string;
user: string;
tenantId?: string;
};
type AvailableProjectFileRecordSource = Pick<
IMongoFile,
| '_id'
| 'file_id'
| 'filename'
| 'filepath'
| 'object'
| 'type'
| 'bytes'
| 'usage'
| 'embedded'
| 'metadata'
| 'context'
| 'user'
| 'tenantId'
>;
function normalizeAvailableProjectFileRecord(
file: AvailableProjectFileRecordSource,
): AvailableProjectFileRecord {
return {
_id: file._id.toString(),
file_id: file.file_id,
filename: file.filename,
filepath: file.filepath,
object: file.object,
type: file.type,
bytes: file.bytes,
usage: file.usage,
...(file.embedded !== undefined ? { embedded: file.embedded } : {}),
...(file.metadata?.searchDestination !== undefined ||
file.metadata?.routingMimeType !== undefined
? {
metadata: {
...(file.metadata.searchDestination !== undefined
? { searchDestination: file.metadata.searchDestination }
: {}),
...(file.metadata.routingMimeType !== undefined
? { routingMimeType: file.metadata.routingMimeType }
: {}),
},
}
: {}),
...(file.context !== undefined ? { context: file.context } : {}),
user: file.user.toString(),
...(file.tenantId !== undefined ? { tenantId: file.tenantId } : {}),
};
}
export type AvailableProjectFilesOptions = FileOwnerScope & {
excludedFileIds?: string[];
limit?: number;
cursor?: string | null;
search?: string;
now?: Date;
};
export type AvailableProjectFilesResult = {
files: AvailableProjectFileRecord[];
nextCursor: string | null;
};
export type ProjectFileRecord = {
_id: string;
file_id: string;
filename: string;
filepath: string;
object: 'file';
type: string;
bytes: number;
usage: number;
embedded?: boolean;
metadata?: { searchDestination?: boolean; routingMimeType?: string };
source?: string;
storageKey?: string;
storageRegion?: string;
context?: string;
expiredAt?: Date | null;
user: string;
tenantId?: string;
createdAt?: Date;
updatedAt?: Date;
previewRevision?: string;
status?: 'pending' | 'ready' | 'failed';
text?: string;
};
type ProjectFileRecordSource = Pick<
IMongoFile,
| '_id'
| 'file_id'
| 'filename'
| 'filepath'
| 'object'
| 'type'
| 'bytes'
| 'usage'
| 'embedded'
| 'metadata'
| 'source'
| 'storageKey'
| 'storageRegion'
| 'context'
| 'expiredAt'
| 'user'
| 'tenantId'
| 'createdAt'
| 'updatedAt'
| 'previewRevision'
| 'status'
| 'text'
>;
function normalizeProjectFileRecord(file: ProjectFileRecordSource): ProjectFileRecord {
return {
_id: file._id.toString(),
file_id: file.file_id,
filename: file.filename,
filepath: file.filepath,
object: file.object,
type: file.type,
bytes: file.bytes,
usage: file.usage,
...(file.embedded !== undefined ? { embedded: file.embedded } : {}),
...(file.metadata?.searchDestination !== undefined ||
file.metadata?.routingMimeType !== undefined
? {
metadata: {
...(file.metadata.searchDestination !== undefined
? { searchDestination: file.metadata.searchDestination }
: {}),
...(file.metadata.routingMimeType !== undefined
? { routingMimeType: file.metadata.routingMimeType }
: {}),
},
}
: {}),
...(file.source !== undefined ? { source: file.source } : {}),
...(file.storageKey !== undefined ? { storageKey: file.storageKey } : {}),
...(file.storageRegion !== undefined ? { storageRegion: file.storageRegion } : {}),
...(file.context !== undefined ? { context: file.context } : {}),
...(file.expiredAt !== undefined ? { expiredAt: file.expiredAt } : {}),
user: file.user.toString(),
...(file.tenantId !== undefined ? { tenantId: file.tenantId } : {}),
...(file.createdAt !== undefined ? { createdAt: file.createdAt } : {}),
...(file.updatedAt !== undefined ? { updatedAt: file.updatedAt } : {}),
...(file.previewRevision !== undefined ? { previewRevision: file.previewRevision } : {}),
...(file.status !== undefined ? { status: file.status } : {}),
...(file.text !== undefined ? { text: file.text } : {}),
};
}
export type ProjectFilesOptions = FileOwnerScope & {
fileIds: string[];
includeContent?: boolean;
};
export class InvalidAvailableProjectFilesCursorError extends Error {
constructor() {
super('Invalid project file cursor');
this.name = 'InvalidAvailableProjectFilesCursorError';
}
}
export function parseAvailableProjectFilesCursor(cursor?: string | null): string | null {
if (cursor == null || cursor === '') {
return null;
}
if (cursor.length !== 24 || !Types.ObjectId.isValid(cursor)) {
throw new InvalidAvailableProjectFilesCursorError();
}
return cursor;
}
export type FileOwnerScope = {
userId: string;
tenantId?: string | null;
};
export type ExpiredFileQueryOptions = {
now?: Date;
};
export class FileStorageConflictError extends Error {
readonly code: string = 'FILE_STORAGE_CONFLICT';
constructor() {
super('The file has a conflicting storage operation');
this.name = 'FileStorageConflictError';
}
}
function codeFileStorageFilter(): FilterQuery<IMongoFile> {
return {
'storageOperations.deletionPending': { $ne: true },
'storageOperations.deleting': { $exists: false },
'storageOperations.provisioning.0': { $exists: false },
};
}
function rethrowCodeFileStorageConflict(error: unknown): never {
if (error instanceof Error && 'code' in error && error.code === 11000) {
throw new FileStorageConflictError();
}
throw error;
}
function assertValidCodeEnvironmentRouteKey(routeKey: string): void {
if (routeKey.length === 0 || routeKey.includes('.') || routeKey.startsWith('$')) {
throw new Error(`Invalid code environment route key "${routeKey}"`);
}
}
function withOwnerScope<T extends FilterQuery<IMongoFile>>(
filter: T,
ownerScope?: FileOwnerScope,
): T & FilterQuery<IMongoFile> {
if (!ownerScope) {
return filter;
}
const scopedFilter: T & FilterQuery<IMongoFile> = {
...filter,
user: ownerScope.userId,
};
if (ownerScope.tenantId) {
scopedFilter.tenantId = ownerScope.tenantId;
}
return scopedFilter;
}
/** Re-provisioned copies keep their parent row and erasure retry anchor. */
function fileStorageCleanupFilter(data: FileStorageCleanupParams): FilterQuery<IMongoFile> {
return {
file_id: data.file_id,
user: data.userId,
$and: [{ tenantId: data.tenantId ?? null }],
updatedAt: data.updatedAt ?? { $exists: false },
'metadata.codeEnvRef': null,
'metadata.codeEnvRefs': { $in: [null, {}] },
'storageOperations.provisioning.0': { $exists: false },
'storageOperations.deleting': { $exists: false },
...(data.source === FileSources.vectordb
? {
embedded: data.embedded ?? { $ne: true },
'metadata.embeddedEntities': data.embeddedEntities ?? null,
}
: { embedded: { $ne: true } }),
};
}
type PlainFileDocument = Omit<TFile, 'user' | '_id'> & {
user: IMongoFile['user'];
_id?: IMongoFile['_id'];
};
type RunArtifactDocument = Omit<RunArtifactFile, 'user' | '_id'> & PlainFileDocument;
function runArtifactFilter(scope: RunArtifactRunScope): FilterQuery<IMongoFile> {
const required = [scope.userId, scope.conversationId, scope.runId];
if (required.some((value) => typeof value !== 'string' || value.trim().length === 0)) {
throw new Error('A complete run artifact owner scope is required');
}
if (scope.tenantId != null && scope.tenantId.trim().length === 0) {
throw new Error('An empty run artifact tenant is invalid');
}
return {
user: scope.userId,
tenantId: scope.tenantId ?? null,
conversationId: scope.conversationId,
context: FileContext.run_artifact,
'metadata.runFile.runId': scope.runId,
};
}
function runArtifactIdentityFilter(scope: RunArtifactScope): FilterQuery<IMongoFile> {
const filter = runArtifactFilter(scope);
if (
[scope.executionId, scope.agentId, scope.sourceFileId].some(
(value) => typeof value !== 'string' || value.trim().length === 0,
)
) {
throw new Error('A complete run artifact execution scope is required');
}
return {
...filter,
'metadata.runFile.executionId': scope.executionId,
'metadata.runFile.agentId': scope.agentId,
'metadata.runFile.sourceFileId': scope.sourceFileId,
};
}
/** A versioned, scoped UUID reserves no incomplete file document or downloadable bytes. */
function runArtifactFileId(scope: RunArtifactScope): string {
const bytes = createHash('sha256')
.update(
JSON.stringify([
'librechat-run-artifact:v1',
scope.userId,
scope.tenantId ?? null,
scope.conversationId,
scope.runId,
scope.executionId,
scope.agentId,
scope.sourceFileId,
]),
)
.digest()
.subarray(0, 16);
bytes[6] = (bytes[6] & 0x0f) | 0x80;
bytes[8] = (bytes[8] & 0x3f) | 0x80;
const hex = bytes.toString('hex');
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
}
function serializeRunArtifact(
file: RunArtifactDocument,
scope?: RunArtifactScope,
): RunArtifactFile {
if (scope != null && file.metadata.runFile.agentId !== scope.agentId) {
throw new Error('The run artifact belongs to a different producing agent');
}
const { _id, user, ...fields } = file;
return { ...fields, user: user.toString(), ...(_id == null ? {} : { _id: String(_id) }) };
}
/** Factory function that takes mongoose instance and returns the file methods */
export function createFileMethods(mongoose: typeof import('mongoose')): {
getRunFileCandidates: (fileIds: readonly string[], tenantId?: string | null) => Promise<TFile[]>;
claimRunArtifactFile: (scope: RunArtifactScope) => Promise<RunArtifactClaim>;
publishRunArtifactFile: (input: PublishRunArtifactInput) => Promise<RunArtifactFile>;
findRunArtifactFile: (scope: RunArtifactScope) => Promise<RunArtifactFile | null>;
listRunArtifacts: (scope: RunArtifactRunScope) => Promise<RunArtifactFile[]>;
findFileById: (file_id: string, options?: Record<string, unknown>) => Promise<IMongoFile | null>;
getFiles: (
filter: FilterQuery<IMongoFile>,
_sortOptions?: Record<string, SortOrder> | null,
selectFields?: Record<string, 0 | 1> | string | null,
limit?: number | null,
) => Promise<IMongoFile[] | null>;
getProjectFiles: (options: ProjectFilesOptions) => Promise<ProjectFileRecord[]>;
getAvailableProjectFiles: (
options: AvailableProjectFilesOptions,
) => Promise<AvailableProjectFilesResult>;
getExpiredFiles: (limit?: number, options?: ExpiredFileQueryOptions) => Promise<IMongoFile[]>;
incrementFileDeletionAttempts: (file_id: string) => Promise<number>;
deferExpiredFile: (file_id: string, deletionRetryAt: Date) => Promise<void>;
getToolFilesByIds: (
fileIds: string[],
toolResourceSet?: Set<EToolResources>,
ownerScope?: FileOwnerScope,
) => Promise<IMongoFile[]>;
getCodeGeneratedFiles: (
conversationId: string,
threadFileIds?: string[],
ownerScope?: FileOwnerScope,
) => Promise<IMongoFile[]>;
getUserCodeFiles: (fileIds: string[], ownerScope: FileOwnerScope) => Promise<IMongoFile[]>;
getDeferredProvisionFiles: (
fileIds: string[],
ownerScope: FileOwnerScope,
resources?: {
code?: boolean;
search?: boolean;
codeRouteKey?: string;
searchNamespaces?: string[];
hydrateProvisioned?: boolean;
},
) => Promise<IMongoFile[]>;
claimCodeFile: (data: {
filename: string;
conversationId: string;
file_id: string;
user: string;
tenantId?: string | null;
sourceDispatchedAt?: number;
}) => Promise<IMongoFile>;
commitCodeFile: (data: CodeFileCommitData, sourceDispatchedAt?: number) => Promise<boolean>;
createFile: (data: Partial<IMongoFile>, disableTTL?: boolean) => Promise<IMongoFile | null>;
updateFile: (
data: Partial<IMongoFile> & { file_id: string },
extraFilter?: FilterQuery<IMongoFile>,
) => Promise<IMongoFile | null>;
updateFileCodeEnvRef: (data: {
file_id: string;
routeKey: string;
ref: CodeEnvRef;
legacyRef?: CodeEnvRef;
}) => Promise<IMongoFile | null>;
clearFileCodeEnvRefs: (data: ClearFileCodeEnvRefsParams) => Promise<boolean>;
clearFileEmbeddings: (data: ClearFileEmbeddingsParams) => Promise<boolean>;
isFileStorageClean: (data: FileStorageCleanupParams) => Promise<boolean>;
addFileEmbeddedEntity: (data: {
file_id: string;
entityId: string;
}) => Promise<IMongoFile | null>;
recordFileWorkspaceImport: (data: WorkspaceImportRecord) => Promise<boolean>;
adoptLegacySearchEmbeddings: (data: {
files: LegacySearchFileRef[];
agentId: string;
}) => Promise<string[]>;
updateFileUsage: (data: {
file_id: string;
inc?: number;
user?: string;
tenantId?: string | null;
}) => Promise<IMongoFile | null>;
deleteFile: (file_id: string) => Promise<IMongoFile | null>;
deleteFiles: (
file_ids: string[],
user?: string,
options?: DeleteFilesOptions,
) => Promise<DeleteFilesResult>;
deleteFileByFilter: (filter: FilterQuery<IMongoFile>) => Promise<IMongoFile | null>;
batchUpdateFiles: (
updates: Array<{
file_id: string;
filepath: string;
storageKey?: string;
storageRegion?: string;
}>,
) => Promise<void>;
updateFilesUsage: (
files: Array<{ file_id: string }>,
fileIds?: string[],
options?: { user?: string; tenantId?: string | null },
) => Promise<IMongoFile[]>;
extendFilesTTL: (
fileIds: string[],
hold: { renewMs: number; maxLifetimeMs: number },
owner: { user: string; tenantId?: string | null },
) => Promise<number>;
sweepOrphanedPreviews: (maxAgeMs?: number) => Promise<number>;
} {
/** Hydrates only host-selected IDs; the caller applies its existing agent-file authorization. */
async function getRunFileCandidates(
fileIds: readonly string[],
tenantId?: string | null,
): Promise<TFile[]> {
if (fileIds.length === 0) {
return [];
}
const File = mongoose.models.File as Model<IMongoFile>;
const files = await File.find({
file_id: { $in: fileIds },
tenantId: tenantId ?? null,
}).lean<PlainFileDocument[]>();
return files.map(({ user, _id, ...file }) => ({
...file,
user: String(user),
embedded: file.embedded === true,
...(_id == null ? {} : { _id: String(_id) }),
}));
}
async function findRunArtifactFile(scope: RunArtifactScope): Promise<RunArtifactFile | null> {
const File = mongoose.models.File as Model<IMongoFile>;
const file = await File.findOne(runArtifactIdentityFilter(scope)).lean<RunArtifactDocument>();
return file == null ? null : serializeRunArtifact(file, scope);
}
async function listRunArtifacts(scope: RunArtifactRunScope): Promise<RunArtifactFile[]> {
const File = mongoose.models.File as Model<IMongoFile>;
const files = await File.find(runArtifactFilter(scope))
.sort({ createdAt: 1, file_id: 1 })
.lean<RunArtifactDocument[]>();
return files.map((file) => serializeRunArtifact(file));
}
async function claimRunArtifactFile(scope: RunArtifactScope): Promise<RunArtifactClaim> {
const file = await findRunArtifactFile(scope);
return file == null ? { file_id: runArtifactFileId(scope) } : { file_id: file.file_id, file };
}
/** Publishes one complete storage result. Retried writers cannot replace its bytes or grants. */
async function publishRunArtifactFile({
scope,
file,
provenance,
}: PublishRunArtifactInput): Promise<RunArtifactFile> {
const File = mongoose.models.File as Model<IMongoFile>;
const filter = runArtifactIdentityFilter(scope);
if (
provenance.runId !== scope.runId ||
provenance.executionId !== scope.executionId ||
provenance.agentId !== scope.agentId ||
provenance.sourceFileId !== scope.sourceFileId ||
!Number.isFinite(Date.parse(provenance.publishedAt))
) {
throw new Error('Run artifact provenance does not match its publication scope');
}
if (
!file.filepath ||
!file.filename ||
!file.type ||
file.source === FileSources.execute_code ||
!Number.isSafeInteger(file.bytes) ||
file.bytes < 0
) {
throw new Error('A durable stored file is required to publish a run artifact');
}
const now = new Date();
const insert = {
filename: file.filename,
filepath: file.filepath,
bytes: file.bytes,
type: file.type,
storageKey: file.storageKey,
storageRegion: file.storageRegion,
text: file.text,
textFormat: file.textFormat,
status: file.status,
previewError: file.previewError,
previewRevision: file.previewRevision,
width: file.width,
height: file.height,
messageId: file.messageId,
expiredAt: file.expiredAt,
file_id: runArtifactFileId(scope),
user: scope.userId,
tenantId: scope.tenantId ?? undefined,
conversationId: scope.conversationId,
context: FileContext.run_artifact,
object: 'file',
source: file.source ?? FileSources.local,
embedded: false,
embeddingOwnersComplete: true,
embeddingOwnersTrackedAt: new Date(),
usage: 1,
llmDeliveryPath: file.llmDeliveryPath ?? 'none',
metadata: {
...file.metadata,
// The source is a private execution object. Consumers must provision the
// immutable durable copy, never reuse that potentially mutable source.
codeEnvRef: undefined,
codeEnvRefs: undefined,
embeddedEntities: undefined,
destinationChosen: false,
runFile: provenance,
},
createdAt: now,
updatedAt: now,
};
try {
const published = await File.findOneAndUpdate(
filter,
{ $setOnInsert: insert },
{ upsert: true, new: true, runValidators: true, timestamps: false },
).lean<RunArtifactDocument>();
if (published == null) {
throw new Error('Run artifact publication did not produce a file');
}
return serializeRunArtifact(published, scope);
} catch (error) {
if (error instanceof Error && 'code' in error && error.code === 11000) {
const existing = await findRunArtifactFile(scope);
if (existing != null) {
return existing;
}
}
throw error;
}
}
/**
* Finds a file by its file_id with additional query options.
* @param file_id - The unique identifier of the file
* @param options - Query options for filtering, projection, etc.
* @returns A promise that resolves to the file document or null
*/
async function findFileById(
file_id: string,
options: Record<string, unknown> = {},
): Promise<IMongoFile | null> {
const File = mongoose.models.File as Model<IMongoFile>;
return File.findOne({ file_id, ...options }).lean<IMongoFile>();
}
/** Select fields for query projection - 0 to exclude, 1 to include */
type SelectProjection = Record<string, 0 | 1>;
/**
* Retrieves files matching a given filter, sorted by the most recently updated.
* @param filter - The filter criteria to apply
* @param _sortOptions - Optional sort parameters
* @param selectFields - Fields to include/exclude in the query results. Default excludes the 'text' field
* @param limit - Optional max number of documents to return (newest first)
* @returns A promise that resolves to an array of file documents
*/
async function getFiles(
filter: FilterQuery<IMongoFile>,
_sortOptions?: Record<string, SortOrder> | null,
selectFields?: string | Record<string, 0 | 1> | null | undefined,
limit?: number | null,
): Promise<IMongoFile[] | null> {
const File = mongoose.models.File as Model<IMongoFile>;
const sortOptions = { updatedAt: -1 as SortOrder, ..._sortOptions };
const query = File.find(filter);
if (selectFields != null) {
query.select(selectFields);
} else {
query.select({ text: 0 });
}
if (limit != null && limit > 0) {
query.limit(limit);
}
return await query.sort(sortOptions).lean<IMongoFile[]>();
}
async function getProjectFiles({
fileIds,
userId,
tenantId,
includeContent = false,
}: ProjectFilesOptions): Promise<ProjectFileRecord[]> {
const File = mongoose.models.File as Model<IMongoFile>;
const query = File.find({
file_id: { $in: fileIds },
user: userId,
$or: [{ embedded: true }, { 'metadata.searchDestination': true }],
context: FileContext.message_attachment,
// Mongo's null predicate matches explicit null and legacy missing tenant fields.
tenantId: tenantId != null && tenantId !== '' ? tenantId : null,
});
if (includeContent) {
query.select({});
} else {
query.select({
_id: 1,
file_id: 1,
filename: 1,
filepath: 1,
object: 1,
type: 1,
bytes: 1,
usage: 1,
embedded: 1,
'metadata.searchDestination': 1,
'metadata.routingMimeType': 1,
source: 1,
storageKey: 1,
storageRegion: 1,
context: 1,
expiredAt: 1,
user: 1,
tenantId: 1,
createdAt: 1,
updatedAt: 1,
previewRevision: 1,
status: 1,
});
}
const files = (await query.lean<ProjectFileRecordSource[]>()) ?? [];
return files.map(normalizeProjectFileRecord);
}
async function getAvailableProjectFiles({
userId,
tenantId,
excludedFileIds = [],
limit = DEFAULT_AVAILABLE_PROJECT_FILES_LIMIT,
cursor,
search,
now = new Date(),
}: AvailableProjectFilesOptions): Promise<AvailableProjectFilesResult> {
if (!Number.isInteger(limit) || limit < 1 || limit > MAX_AVAILABLE_PROJECT_FILES_LIMIT) {
throw new RangeError('Invalid project file limit');
}
const decodedCursor = parseAvailableProjectFilesCursor(cursor);
const filters: FilterQuery<IMongoFile>[] = [
{
user: userId,
tenantId: tenantId != null && tenantId !== '' ? tenantId : null,
$or: [{ embedded: true }, { 'metadata.searchDestination': true }],
context: FileContext.message_attachment,
$and: [{ $or: [{ expiredAt: null }, { expiredAt: { $gt: now } }] }],
},
];
if (excludedFileIds.length > 0) {
filters[0].file_id = { $nin: excludedFileIds };
}
const normalizedSearch = search?.trim();
if (normalizedSearch) {
filters.push({
filename: { $regex: escapeRegExp(normalizedSearch), $options: 'i' },
});
}
if (decodedCursor) {
filters.push({ _id: { $lt: new Types.ObjectId(decodedCursor) } });
}
const File = mongoose.models.File as Model<IMongoFile>;
const files = await File.find(filters.length === 1 ? filters[0] : { $and: filters })
.select({
_id: 1,
file_id: 1,
filename: 1,
filepath: 1,
object: 1,
type: 1,
bytes: 1,
usage: 1,
embedded: 1,
'metadata.searchDestination': 1,
'metadata.routingMimeType': 1,
context: 1,
user: 1,
tenantId: 1,
})
.sort({ _id: -1 })
.limit(limit + 1)
.lean<AvailableProjectFileRecordSource[]>();
let nextCursor: string | null = null;
if (files.length > limit) {
files.pop();
const lastFile = files[files.length - 1];
if (lastFile) {
nextCursor = lastFile._id.toString();
}
}
return { files: files.map(normalizeAvailableProjectFileRecord), nextCursor };
}
/**
* Expired files the sweep may attempt right now, oldest deadline first.
*
* `deletionRetryAt` is the only thing holding a file back: one that keeps
* failing is deferred on a growing backoff, and one that exhausts its
* attempts is parked far enough out to stop crowding the batch. Without
* it a permanently undeletable file sorts to the front of this bounded
* batch on every pass and starves every file that expired after it.
*
* The deferral is deliberately a deadline rather than a flag, so nothing
* here is ever excluded for good. File records are reused across content
* lifecycles — a code-output row is repurposed for a repeated
* `(filename, conversationId)`, keeping fields it was not asked to change
* — so bookkeeping that permanently excluded a row would eventually
* strand a *different* object than the one it was recorded against. A
* deadline can only ever delay that; it cannot lose it.
*
* An absent field means "never attempted", so records written before it
* existed remain eligible.
*/
async function getExpiredFiles(
limit = 100,
{ now = new Date() }: ExpiredFileQueryOptions = {},
): Promise<IMongoFile[]> {
const File = mongoose.models.File as Model<IMongoFile>;
return await File.find({
expiredAt: { $ne: null, $lte: now },
$or: [{ deletionRetryAt: null }, { deletionRetryAt: { $lte: now } }],
})
.sort({ expiredAt: 1 })
.limit(limit)
.lean<IMongoFile[]>();
}
/**
* Records one failed sweep deletion and returns the file's resulting
* consecutive-failure count.
*
* The count comes back from the increment itself rather than being
* re-derived from the caller's snapshot. Two nodes sweeping the same file
* would otherwise each read the same value and each believe itself to be
* the same attempt, pushing the stored count past the give-up cap while
* both still think it is below — so the give-up would never be reported.
* Returning it here gives every caller a distinct attempt number.
*/
async function incrementFileDeletionAttempts(file_id: string): Promise<number> {
const File = mongoose.models.File as Model<IMongoFile>;
const file = await File.findOneAndUpdate(
{ file_id },
{ $inc: { deletionAttempts: 1 } },
/** `timestamps: false`: sweep bookkeeping is not a content write.
* `processCodeOutput` falls back to `updatedAt` as the writer-order
* stamp for records predating `metadata.sourceDispatchedAt`, so
* bumping it here would make a failed sweep look like a newer writer
* and a background harvest would drop its attachment. Same reasoning
* as `claimCodeFile`. */
{ new: true, projection: { deletionAttempts: 1 }, timestamps: false },
).lean<Pick<IMongoFile, 'deletionAttempts'> | null>();
return file?.deletionAttempts ?? 0;
}
/**
* Holds a file back from the sweep until `deletionRetryAt`.
*
* Written with `$max` so a deferral can only ever move later. A node that
* computed a shorter backoff from a lower attempt count cannot pull the
* file forward past a longer one another node already committed.
*/
async function deferExpiredFile(file_id: string, deletionRetryAt: Date): Promise<void> {
const File = mongoose.models.File as Model<IMongoFile>;
await File.updateOne({ file_id }, { $max: { deletionRetryAt } }, { timestamps: false }).exec();
}
/**
* Retrieves tool files (files that are embedded or have a fileIdentifier) from an array of file IDs.
* Note: execute_code files are handled separately by getCodeGeneratedFiles.
* @param fileIds - Array of file_id strings to search for
* @param toolResourceSet - Optional filter for tool resources
* @returns Files that match the criteria
*/
async function getToolFilesByIds(
fileIds: string[],
toolResourceSet?: Set<EToolResources>,
ownerScope?: FileOwnerScope,
): Promise<IMongoFile[]> {
if (!fileIds || !fileIds.length || !toolResourceSet?.size) {
return [];
}
try {
const orConditions: FilterQuery<IMongoFile>[] = [];
if (toolResourceSet.has(EToolResources.context)) {
orConditions.push({ text: { $exists: true, $ne: null }, context: FileContext.agents });
}
if (toolResourceSet.has(EToolResources.file_search)) {
orConditions.push({ embedded: true });
}
// If no conditions to match, return empty
if (orConditions.length === 0) {
return [];
}
const filter = withOwnerScope(
{
file_id: { $in: fileIds },
context: { $ne: FileContext.execute_code },
$or: orConditions,
},
ownerScope,
);
const selectFields: SelectProjection = { text: 0 };
const sortOptions = { updatedAt: -1 as SortOrder };
const results = await getFiles(filter, sortOptions, selectFields);
return results ?? [];
} catch (error) {
logger.error('[getToolFilesByIds] Error retrieving tool files:', error);
throw new Error('Error retrieving tool files');
}
}
/**
* Retrieves files generated by code execution for a given conversation.
* These files are stored locally with `metadata.codeEnvRef` for code env re-upload.
*
* Filtered by `file_id IN threadFileIds` (the file_ids referenced from
* messages in the current thread's `files` arrays) rather than by
* `messageId IN threadMessageIds` (the File record's own creator id).
* This mirrors `getUserCodeFiles` and fixes the branched-conversation
* case where:
* 1. Sibling A creates `output.csv` via processCodeOutput → File
* record persists with `messageId = A`.
* 2. Sibling N (a regeneration of A's parent) recreates `output.csv`.
* `claimCodeFile` is keyed by `(filename, conversationId, context)`
* and finds A's record; `processCodeOutput` deliberately preserves
* A's `messageId` to keep file→original-creator provenance intact.
* 3. Turn N+1's user message has `parentMessageId = N` (sibling of A).
* `getThreadData` walks back from N: thread = [N, root]. A is
* NOT in that thread, so a `messageId IN [N, root]` filter
* excludes the file.
* The message at N references the file via `messages.files = [...]`,
* so `threadFileIds` (collected from that array during thread walk)
* DOES include the file_id. Filtering by that key reaches the right
* file regardless of which sibling first created it.
*
* @param conversationId - The conversation ID to search for
* @param threadFileIds - Array of file_ids referenced by messages in
* the current thread (collected from `messages.files[].file_id`).
* While technically optional, this function returns empty if not
* provided — code-generated files must be filtered by thread to
* avoid pulling in files from other branches of a conversation.
* @returns Files generated by code execution in the conversation,
* filtered to those referenced by the current thread.
*/
async function getCodeGeneratedFiles(
conversationId: string,
threadFileIds?: string[],
ownerScope?: FileOwnerScope,
): Promise<IMongoFile[]> {
if (!conversationId) {
return [];
}
/**
* Return early if threadFileIds not provided — same rationale as
* the messageIds-based predecessor: code-generated files must be
* filtered by thread to ensure we only return files relevant to
* the current conversation branch, not orphaned files from other
* branches or deleted messages.
*/
if (!threadFileIds || threadFileIds.length === 0) {
return [];
}
try {
const filter = withOwnerScope(
{
conversationId,
context: FileContext.execute_code,
file_id: { $in: threadFileIds },
$or: [
{ 'metadata.codeEnvRef': { $exists: true } },
{ 'metadata.codeEnvRefs': { $exists: true } },
{ 'metadata.codeDestination': true },
],
},
ownerScope,
);
const selectFields: SelectProjection = { text: 0 };
const sortOptions = { createdAt: 1 as SortOrder };
const results = await getFiles(filter, sortOptions, selectFields);
return results ?? [];
} catch (error) {
logger.error('[getCodeGeneratedFiles] Error retrieving code generated files:', error);
return [];
}
}
/**
* Retrieves conversation attachments that were accepted but never provisioned, so a
* later turn can still queue them.
*
* Lazy provisioning defers the upload to the sandbox or vector store until a tool
* actually runs. The other hydration queries only return files that already carry
* the result of that work: `getToolFilesByIds` matches `embedded: true` for
* file_search, and `getUserCodeFiles` requires an existing `codeEnvRef`. A file
* whose tool was never called on its upload turn therefore satisfies neither and
* disappears. This fills exactly that gap: attachments with no code reference and
* no embedding.
*
* Kept separate from delivery hydration deliberately. These records are candidates
* for provisioning only; feeding them back into the model's attachments would
* re-send earlier uploads on every subsequent turn.
*
* Selection is per requested resource, not per file. A file embedded for an earlier
* file_search agent still has no code reference, so a later execute_code agent must
* see it; requiring both results to be absent would hide exactly that case.
*
* @param fileIds - Candidate file IDs from the current thread
* @param ownerScope - Authenticated owner scope
* @param resources - Which provisioning results the current agent needs
* @returns Attachments still awaiting a result the current agent needs
*/
async function getDeferredProvisionFiles(
fileIds: string[],
ownerScope: FileOwnerScope,
resources: {
code?: boolean;
search?: boolean;
codeRouteKey?: string;
searchNamespaces?: string[];
/** Set when no other query hydrates already-provisioned files this turn. */
hydrateProvisioned?: boolean;
} = {
code: true,
search: true,
},
): Promise<IMongoFile[]> {
if (!fileIds || fileIds.length === 0) {
return [];
}
const missingConditions: FilterQuery<IMongoFile>[] = [];
/* When no other query hydrates this turn's files, every eligible record has to be
* loaded, not only the unprovisioned ones: priming adds the provisioned ones to the
* tool resources, and the code probe screens a default-route session for liveness.
* Left behind, a file the previous turn provisioned successfully is the one that
* goes missing, while one that failed is retried. This holds for search as much as
* for code: an attachment embedded on an earlier turn is hydrated by nothing else,
* so the next search runs with no reference to it. */
const hydrateEverything =
resources.hydrateProvisioned === true &&
(resources.code === true || resources.search === true);
if (resources.code && !hydrateEverything) {
/* A reference for another deployment does not make the file usable here, and with
* resendFiles off nothing downstream re-reads the record to notice, so eligibility
* is judged against the route this turn will actually execute on. The legacy
* pointer counts only when it resolves to that same route, mirroring how
* mergeCodeEnvRef keys it: executionRouteKey, then executionProfile, then default. */
const routeKey = resources.codeRouteKey ?? 'default';
assertValidCodeEnvironmentRouteKey(routeKey);
const usableForRoute: FilterQuery<IMongoFile>[] = [
{ [`metadata.codeEnvRefs.${routeKey}`]: { $exists: true } },
{ 'metadata.codeEnvRef.executionRouteKey': routeKey },
{
'metadata.codeEnvRef': { $exists: true },
'metadata.codeEnvRef.executionRouteKey': { $exists: false },
'metadata.codeEnvRef.executionProfile': routeKey,
},
];
if (routeKey === 'default') {
usableForRoute.push({
'metadata.codeEnvRef': { $exists: true },
'metadata.codeEnvRef.executionRouteKey': { $exists: false },
'metadata.codeEnvRef.executionProfile': { $in: [null, 'default'] },
});
}
missingConditions.push({ $nor: usableForRoute });
}
if (resources.search && !hydrateEverything) {
/* The record-wide flag only says the file was embedded somewhere, so for a record
* whose vectors live in an agent namespace it cannot answer whether the namespace
* this turn searches has them. Membership is not known here, so a record missing
* from any candidate namespace is loaded and judged later, where it is. */
const namespaces = resources.searchNamespaces ?? [];
if (namespaces.length === 0) {
missingConditions.push({ embedded: { $ne: true } });
} else {
missingConditions.push({
$or: [
{ context: { $ne: FileContext.agents }, embedded: { $ne: true } },
...namespaces.map((namespace) => ({
context: FileContext.agents,
'metadata.embeddedEntities': { $ne: namespace },
})),
],
});
}
}
if (!hydrateEverything && missingConditions.length === 0) {
return [];
}
try {
const filter = withOwnerScope(
{
file_id: { $in: fileIds },
context: { $ne: FileContext.execute_code },
...(hydrateEverything ? {} : { $or: missingConditions }),
},
ownerScope,
);
const selectFields: SelectProjection = { text: 0 };
const sortOptions = { createdAt: 1 as SortOrder };
const results = await getFiles(filter, sortOptions, selectFields);
return results ?? [];
} catch (error) {
/* An empty list here is indistinguishable from nothing needing provisioning, so the
* turn would build no provisioning state and let the tool run without the
* attachment. The callback aborts on missing inputs precisely to avoid that, so a
* read failure has to surface rather than be flattened into a benign answer. */
logger.error('[getDeferredProvisionFiles] Error retrieving deferred files:', error);
throw error;
}
}
/**
* Retrieves user-uploaded execute_code files (not code-generated) by their file IDs.
* These are files with fileIdentifier metadata but context is NOT execute_code (e.g., agents or message_attachment).
* File IDs should be collected from message.files arrays in the current thread.
* @param fileIds - Array of file IDs to fetch (from message.files in the thread)
* @param ownerScope - Authenticated owner scope used to constrain historical refs
* @returns User-uploaded execute_code files
*/
async function getUserCodeFiles(
fileIds: string[],
ownerScope: FileOwnerScope,
): Promise<IMongoFile[]> {
if (!fileIds || fileIds.length === 0) {
return [];
}
try {
const filter = withOwnerScope(
{
file_id: { $in: fileIds },
context: { $ne: FileContext.execute_code },
$or: [
{ 'metadata.codeEnvRef': { $exists: true } },
{ 'metadata.codeEnvRefs': { $exists: true } },
{ 'metadata.codeDestination': true },
],
},
ownerScope,
);
const selectFields: SelectProjection = { text: 0 };
const sortOptions = { createdAt: 1 as SortOrder };
const results = await getFiles(filter, sortOptions, selectFields);
return results ?? [];
} catch (error) {
logger.error('[getUserCodeFiles] Error retrieving user code files:', error);
return [];
}
}
/**
* Atomically claims a file_id for a code-execution output by compound key.
* Uses $setOnInsert so concurrent calls for the same (filename, conversationId)
* converge on a single record instead of creating duplicates.
*/
async function claimCodeFile(data: {
filename: string;
conversationId: string;
file_id: string;
user: string;
tenantId?: string | null;
/** The claimant's dispatch-order stamp, persisted on INSERT so a
* freshly claimed (not-yet-written) row still carries an ownership
* signal for the background harvest's stale-output guard. */
sourceDispatchedAt?: number;
}): Promise<IMongoFile> {
const File = mongoose.models.File as Model<IMongoFile>;
const tenantFilter = data.tenantId ? { tenantId: data.tenantId } : { tenantId: null };
const insertData = {
embeddingOwnersComplete: true,
embeddingOwnersTrackedAt: new Date(),
file_id: data.file_id,
user: data.user,
...(data.tenantId ? { tenantId: data.tenantId } : {}),
...(data.sourceDispatchedAt != null
? { metadata: { sourceDispatchedAt: data.sourceDispatchedAt } }
: {}),
};
let result: IMongoFile | null;
try {
result = await File.findOneAndUpdate(
{
filename: data.filename,
conversationId: data.conversationId,
context: FileContext.execute_code,
...tenantFilter,
...codeFileStorageFilter(),
},
{ $setOnInsert: insertData },
/** `timestamps: false`: a claim is an id reservation, not a content
* write — bumping `updatedAt` here would make the row look freshly
* written to the background harvest's out-of-order guard, which
* compares `updatedAt` against the harvest's start time. */
{ upsert: true, new: true, timestamps: false },
).lean<IMongoFile>();
} catch (error) {
rethrowCodeFileStorageConflict(error);
}
if (!result) {
throw new Error(
`[claimCodeFile] Failed to claim file "${data.filename}" for conversation ${data.conversationId}`,
);
}
return result;
}
/**
* Creates a new file with a TTL of 1 hour.
* @param data - The file data to be created, must contain file_id
* @param disableTTL - Whether to disable the TTL
* @returns A promise that resolves to the created file document
*/
async function createFile(
data: Partial<IMongoFile> | CodeFileCommitData,
disableTTL?: boolean,
): Promise<IMongoFile | null> {
const File = mongoose.models.File as Model<IMongoFile>;
const fileData: (Partial<IMongoFile> | CodeFileCommitData) & { expiresAt?: Date } = {
...data,
expiresAt: new Date(Date.now() + 3600 * 1000),
};
if (disableTTL) {
delete fileData.expiresAt;
}
const generatedOutput = data.context === FileContext.execute_code;
try {
return await File.findOneAndUpdate(
{
file_id: data.file_id,
...(generatedOutput
? {
...codeFileStorageFilter(),
user: data.user,
tenantId: data.tenantId ?? null,
}
: {}),
},
{
...fileContentUpdate(fileData),
$setOnInsert: { embeddingOwnersComplete: true, embeddingOwnersTrackedAt: new Date() },
},
{ new: true, upsert: true },
).lean<IMongoFile>();
} catch (error) {
if (generatedOutput) rethrowCodeFileStorageConflict(error);
throw error;
}
}
/**
* Updates a file identified by file_id with new data and removes the TTL.
*
* `extraFilter` extends the by-id query with additional conditions
* (e.g. `{ previewRevision: '<expected uuid>' }`). When provided, the
* update is conditional — it commits only if the document still
* matches the extra filter, returning `null` otherwise. Used by the
* deferred-preview render to guard against an older render of the
* same `file_id` overwriting a newer turn's record on cross-turn
* filename reuse.
*
* Content timestamps advance only when the caller actually writes a
* field. Clearing the upload TTL alone is bookkeeping: image reuse
* calls this with nothing but the id on every turn, and consumers that
* fingerprint content by `updatedAt` (Project resume compatibility)
* would otherwise read a new version and reject an approved turn.
*
* @param data - The data to update, must contain file_id
* @param extraFilter - Optional extra equality filter merged into the query.
* @returns A promise that resolves to the updated file document, or
* null if the conditional filter excluded it (or the file was
* deleted).
*/
async function updateFile(
data: (Partial<IMongoFile> & { file_id: string }) | CodeFileCommitData,
extraFilter?: FilterQuery<IMongoFile>,
): Promise<IMongoFile | null> {
const File = mongoose.models.File as Model<IMongoFile>;
const { file_id, ...update } = data;
const updateOperation = {
...fileContentUpdate(update),
$unset: { expiresAt: '' },
};
const query: FilterQuery<IMongoFile> = extraFilter ? { file_id, ...extraFilter } : { file_id };
return File.findOneAndUpdate(query, updateOperation, {
new: true,
timestamps: Object.keys(update).length > 0,
}).lean<IMongoFile>();
}
/** Background outputs commit only while their dispatch still owns the claimed filename. */
async function commitCodeFile(
data: CodeFileCommitData,
sourceDispatchedAt?: number,
): Promise<boolean> {
if (sourceDispatchedAt == null) {
await createFile(data, true);
return true;
}
const committed = await updateFile(data, {
...codeFileStorageFilter(),
user: data.user,
tenantId: data.tenantId ?? null,
$or: [
{ 'metadata.sourceDispatchedAt': { $exists: false } },
{ 'metadata.sourceDispatchedAt': { $lte: sourceDispatchedAt } },
],
});
return committed != null;
}
/**
* Records one code-environment route pointer without rewriting the rest of `metadata`.
* Agents provisioning the same file to different deployments write concurrently, and a
* whole-object `$set` built from each caller's pre-provisioning snapshot would drop the
* sibling route that landed in between.
*
* @param data - The file, its route key, the pointer to store, and an optional legacy pointer
* @returns A promise that resolves to the updated file document, or null when absent
*/
async function updateFileCodeEnvRef(data: {
file_id: string;
routeKey: string;
ref: CodeEnvRef;
legacyRef?: CodeEnvRef;
}): Promise<IMongoFile | null> {
const { file_id, routeKey, ref, legacyRef } = data;
/* Route keys become dotted update paths, so a key carrying `.` or a leading `$` would
* write somewhere other than the intended entry. They come from server config, which
* makes a malformed one a configuration error worth surfacing. */
assertValidCodeEnvironmentRouteKey(routeKey);
const File = mongoose.models.File as Model<IMongoFile>;
const update: Record<string, CodeEnvRef> = {
[`metadata.codeEnvRefs.${routeKey}`]: ref,
};
if (legacyRef) {
update['metadata.codeEnvRef'] = legacyRef;
}
return File.findOneAndUpdate(
{ file_id },
{ $set: update, $unset: { expiresAt: '' } },
{ new: true },
).lean<IMongoFile>();
}
/** Confirms targeted routes are absent after reconciling deleted snapshots. */
async function clearFileCodeEnvRefs(data: ClearFileCodeEnvRefsParams): Promise<boolean> {
const File = mongoose.models.File as Model<IMongoFile>;
/* Tenant middleware adds its own predicate; it must not replace the snapshot scope. */
const query: FilterQuery<IMongoFile> = {
file_id: data.file_id,
user: data.userId,
$and: [{ tenantId: data.tenantId ?? null }],
};
/* Compare contract fields individually: BSON object equality depends on field order. */
const matchesRef = (path: string, ref: CodeEnvRef): FilterQuery<IMongoFile> => {
const fields = {
kind: ref.kind,
id: ref.id,
storage_session_id: ref.storage_session_id,
file_id: ref.file_id,
executionProfile: ref.executionProfile,
executionRouteKey: ref.executionRouteKey,
provisionedAt: ref.provisionedAt,
sandboxFilename: ref.sandboxFilename,
version: 'version' in ref ? ref.version : undefined,
};
const match: FilterQuery<IMongoFile> = {};
for (const [field, value] of Object.entries(fields)) {
match[`${path}.${field}`] =
value === undefined ? { $exists: false } : { $eq: value, $exists: true };
}
return match;
};
const matchesByPath = new Map<string, FilterQuery<IMongoFile>[]>();
const addMatch = (path: string, ref: CodeEnvRef): void => {
const matches = matchesByPath.get(path) ?? [];
matches.push(matchesRef(path, ref));
matchesByPath.set(path, matches);
};
for (const { routeKey, ref } of data.refs) {
assertValidCodeEnvironmentRouteKey(routeKey);
addMatch(`metadata.codeEnvRefs.${routeKey}`, ref);
if ((ref.executionRouteKey ?? ref.executionProfile ?? 'default') === routeKey) {
addMatch('metadata.codeEnvRef', ref);
}
}
const results = await Promise.allSettled(
[...matchesByPath].map(async ([path, matches]) =>
File.updateOne(
{ ...query, $or: matches },
{ $unset: { [path]: '' }, $set: { 'metadata.codeDestination': true } },
{ timestamps: false },
),
),
);
for (const result of results) {
if (result.status === 'rejected') {
throw result.reason;
}
}
const stored = await File.findOne(query, {
_id: 0,
'metadata.codeEnvRef': 1,
'metadata.codeEnvRefs': 1,
}).lean<Pick<IMongoFile, 'metadata'>>();
if (!stored) {
return false;
}
const remaining = getCodeEnvRefs(stored.metadata);
if (data.refs.length === 0) {
return remaining.length === 0;
}
const routes = new Set(data.refs.map(({ routeKey }) => routeKey));
return !remaining.some(([routeKey]) => routes.has(routeKey));
}
/** Clears only the embedding snapshot whose vector deletion was confirmed. */
async function clearFileEmbeddings(data: ClearFileEmbeddingsParams): Promise<boolean> {
const File = mongoose.models.File as Model<IMongoFile>;
const start =
data.ownershipTrackingStart == null ? NaN : Date.parse(data.ownershipTrackingStart);
const now = new Date();
const ownership =
Number.isFinite(start) && start <= now.getTime()
? [
{
embedded: true,
embeddingOwnersComplete: true,
embeddingOwnersTrackedAt: { $gte: new Date(start), $lte: now },
'metadata.embeddedEntities': data.embeddedEntities ?? null,
},
]
: [];
const query: FilterQuery<IMongoFile> = {
file_id: data.file_id,
user: data.userId,
$and: [{ tenantId: data.tenantId ?? null }],
updatedAt: data.updatedAt ?? { $exists: false },
$or: [...ownership, { embedded: false, 'metadata.embeddedEntities': { $exists: false } }],
};
const result = await File.updateOne(
query,
{
$set: { embedded: false, 'metadata.searchDestination': true },
$unset: { 'metadata.embeddedEntities': '' },
},
{ timestamps: false },
);
return result.matchedCount === 1;
}
/** Checks the final content/presence snapshot before removing primary storage. */
async function isFileStorageClean(data: FileStorageCleanupParams): Promise<boolean> {
const File = mongoose.models.File as Model<IMongoFile>;
const filter = fileStorageCleanupFilter(data);
delete filter['storageOperations.deleting'];
if ((await File.exists(filter)) != null) return true;
if (
!data.allowMissingFileRecord ||
(data.source !== FileSources.openai && data.source !== FileSources.azure)
) {
return false;
}
return (
(await File.exists({
file_id: data.file_id,
user: data.userId,
$and: [{ tenantId: data.tenantId ?? null }],
})) == null
);
}
/**
* Records a workspace copy of a user's file and/or the digest of the stored bytes, without
* rewriting sibling targets another turn wrote in between. A digest is kept even when the
* copy fails, so a refused file is not hashed again every turn. The target key becomes a
* dotted path, so only an opaque hex digest is accepted; the owner filter keeps a caller
* from marking another user's record.
*
* @returns Whether a record matched.
*/
async function recordFileWorkspaceImport({
file_id,
user,
tenantId,
copy,
digest,
}: WorkspaceImportRecord): Promise<boolean> {
if (copy != null && !/^[a-f0-9]{32,64}$/.test(copy.key)) {
throw new Error('Workspace import keys must be hex digests');
}
const update: Record<
string,
TFileWorkspaceImport | NonNullable<TFileWorkspaceImports['digest']>
> = {};
if (copy != null) {
update[`metadata.workspaceImports.targets.${copy.key}`] = copy.target;
}
if (digest != null) {
update['metadata.workspaceImports.digest'] = digest;
}
if (Object.keys(update).length === 0) {
return false;
}
const File = mongoose.models.File as Model<IMongoFile>;
const result = await File.updateOne(
{ file_id, user, tenantId: tenantId ?? null },
{ $set: update },
/** Bookkeeping, not a content change: project resource versions include `updatedAt`. */
{ timestamps: false },
);
return result.matchedCount > 0;
}
/**
* Records that a file has been embedded into one vector namespace, without disturbing
* the namespaces already recorded. Agents that share a file record, as a duplicate does
* with its source, each need their own embedding.
*
* @param data - The file and the entity whose namespace now holds its vectors
* @returns A promise that resolves to the updated file document, or null when absent
*/
async function addFileEmbeddedEntity(data: {
file_id: string;
entityId: string;
}): Promise<IMongoFile | null> {
const File = mongoose.models.File as Model<IMongoFile>;
return File.findOneAndUpdate(
{ file_id: data.file_id },
{
$set: { embedded: true },
$addToSet: { 'metadata.embeddedEntities': data.entityId },
$unset: { expiresAt: '' },
},
{ new: true },
).lean<IMongoFile>();
}
/**
* Records an agent's namespace on embedded files whose vectors the caller has confirmed are
* in it, keeping any namespaces already recorded. Each record is matched by its owner and
* tenant as well as its `file_id`, which is not unique on its own.
*
* @param data - Verified file records, and the agent whose namespace holds their vectors
* @returns The file ids that now record the agent's namespace
*/
async function adoptLegacySearchEmbeddings(data: {
files: LegacySearchFileRef[];
agentId: string;
}): Promise<string[]> {
if (data.files.length === 0) {
return [];
}
const File = mongoose.models.File as Model<IMongoFile>;
/* An untenanted record is matched as untenanted: omitting the predicate would let a
* same-id record under a tenant match instead. Each write settles on its own, so a
* committed one is reported even when a sibling fails. */
const settled = await Promise.allSettled(
data.files.map((file) =>
File.findOneAndUpdate(
{
file_id: file.file_id,
user: file.user,
tenantId: file.tenantId ?? null,
context: FileContext.agents,
embedded: true,
},
{ $addToSet: { 'metadata.embeddedEntities': data.agentId } },
{ projection: { file_id: 1 } },
).lean<Pick<IMongoFile, 'file_id'>>(),
),
);
const adopted: string[] = [];
settled.forEach((result, index) => {
if (result.status === 'rejected') {
logger.error('[adoptLegacySearchEmbeddings] Failed to record a confirmed embedding', {
file_id: data.files[index].file_id,
error: (result.reason as Error)?.message,
});
} else if (result.value) {
adopted.push(result.value.file_id);
}
});
return adopted;
}
/**
* Increments the usage of a file identified by file_id.
* @param data - The data to update, must contain file_id and the increment value for usage
* @returns A promise that resolves to the updated file document
*/
async function updateFileUsage(data: {
file_id: string;
inc?: number;
user?: string;
tenantId?: string | null;
}): Promise<IMongoFile | null> {
const File = mongoose.models.File as Model<IMongoFile>;
const { file_id, inc = 1, user, tenantId } = data;
const updateOperation = {
$inc: { usage: inc },
$unset: { expiresAt: '', temp_file_id: '' },
};
// Owner scoping is fail-closed: mismatches leave usage and TTL metadata unchanged.
const query: FilterQuery<IMongoFile> = user
? withOwnerScope({ file_id }, { userId: user, tenantId })
: { file_id };
/** Usage and temporary-upload cleanup are bookkeeping, not content writes. */
return File.findOneAndUpdate(query, updateOperation, {
new: true,
timestamps: false,
}).lean<IMongoFile>();
}
/**
* Removes deleted file ids from the owner's Chat Projects so a project never
* keeps a dangling reference, bumping the revision of every project it changes.
*/
async function detachFilesFromProjects(
files: { file_id: string; user: { toString(): string }; tenantId?: string | null }[],
): Promise<void> {
const ChatProject = mongoose.models.ChatProject as Model<IChatProjectDocument> | undefined;
if (!ChatProject || files.length === 0) {
return;
}
const idsByScope = new Map<
string,
{ user: string; tenantId: string | null; fileIds: string[] }
>();
for (const file of files) {
const user = file.user.toString();
const tenantId = file.tenantId ?? null;
const key = JSON.stringify([user, tenantId]);
const scope = idsByScope.get(key);
if (scope) {
scope.fileIds.push(file.file_id);
} else {
idsByScope.set(key, { user, tenantId, fileIds: [file.file_id] });
}
}
try {
await Promise.all(
[...idsByScope.values()].map(({ user, tenantId, fileIds }) =>
ChatProject.updateMany(
{ user, $and: [{ tenantId }], file_ids: { $in: fileIds } },
{ $pull: { file_ids: { $in: fileIds } }, $inc: { contextRevision: 1 } },
),
),
);
} catch (error) {
logger.warn('[detachFilesFromProjects] Failed to detach deleted files from projects', error);
}
}
/** Clears every project file reference of a user whose files were all deleted. */
async function detachAllUserFilesFromProjects(user: string): Promise<void> {
const ChatProject = mongoose.models.ChatProject as Model<IChatProjectDocument> | undefined;
if (!ChatProject) {
return;
}
try {
await ChatProject.updateMany(
{ user, 'file_ids.0': { $exists: true } },
{ $set: { file_ids: [] }, $inc: { contextRevision: 1 } },
);
} catch (error) {
logger.warn('[detachAllUserFilesFromProjects] Failed to detach deleted files', error);
}
}
/**
* Deletes a file identified by file_id.
* @param file_id - The unique identifier of the file to delete
* @returns A promise that resolves to the deleted file document or null
*/
async function deleteFile(file_id: string): Promise<IMongoFile | null> {
const File = mongoose.models.File as Model<IMongoFile>;
const deleted = await File.findOneAndDelete({ file_id }).lean<IMongoFile>();
if (deleted) {
await detachFilesFromProjects([deleted]);
}
return deleted;
}
/**
* Deletes a file identified by a filter.
* @param filter - The filter criteria to apply
* @returns A promise that resolves to the deleted file document or null
*/
async function deleteFileByFilter(filter: FilterQuery<IMongoFile>): Promise<IMongoFile | null> {
const File = mongoose.models.File as Model<IMongoFile>;
const deleted = await File.findOneAndDelete(filter).lean<IMongoFile>();
if (deleted) {
await detachFilesFromProjects([deleted]);
}
return deleted;
}
/**
* Deletes multiple files identified by an array of file_ids.
* @param file_ids - The unique identifiers of the files to delete
* @param user - Optional user ID to filter by
* @returns A promise that resolves to the result of the deletion operation
*/
async function deleteFiles(
file_ids: string[],
user?: string,
options?: DeleteFilesOptions,
): Promise<DeleteFilesResult> {
const File = mongoose.models.File as Model<IMongoFile>;
if (options) {
const ids = new Set(file_ids);
if (ids.size === 0) return { deletedCount: 0, deletedFileIds: [], failedFileIds: [] };
const snapshots = new Map(options.files.map((file) => [file.file_id, file]));
const scopes = [...ids].map((id) => {
const file = snapshots.get(id);
if (!file) throw new Error('Missing file deletion snapshot');
return {
file_id: id,
userId: file.user?.toString() ?? options.userId,
tenantId: file.tenantId,
updatedAt: file.updatedAt,
source: file.source,
embedded: file.embedded,
embeddedEntities: file.metadata?.embeddedEntities,
};
});
const deleteQuery = { $or: scopes.map(fileStorageCleanupFilter) };
const candidates = await File.find(deleteQuery)
.select({ _id: 1, file_id: 1, user: 1, tenantId: 1 })
.lean<Pick<IMongoFile, '_id' | 'file_id' | 'user' | 'tenantId'>[]>();
const result = await File.deleteMany({
...deleteQuery,
_id: { $in: candidates.map((file) => file._id) },
});
const remaining = await File.find({
$or: scopes.map((scope) => ({
file_id: scope.file_id,
user: scope.userId,
$and: [{ tenantId: scope.tenantId ?? null }],
})),
})
.select({ file_id: 1 })
.lean<Pick<IMongoFile, 'file_id'>[]>();
const remainingIds = new Set(remaining.map((file) => file.file_id));
const deletedFileIds: string[] = [];
const failedFileIds: string[] = [];
const detached = [];
for (const scope of scopes) {
if (remainingIds.has(scope.file_id)) {
failedFileIds.push(scope.file_id);
} else {
deletedFileIds.push(scope.file_id);
detached.push({ file_id: scope.file_id, user: scope.userId, tenantId: scope.tenantId });
}
}
await detachFilesFromProjects(detached);
return { deletedCount: result.deletedCount, deletedFileIds, failedFileIds };
}
if (user) {
const result = await File.deleteMany({ user });
await detachAllUserFilesFromProjects(user);
return result;
}
const deleteQuery: FilterQuery<IMongoFile> = { file_id: { $in: file_ids } };
const doomed = await File.find(deleteQuery)
.select({ file_id: 1, user: 1, tenantId: 1 })
.lean<IMongoFile[]>();
const result = await File.deleteMany(deleteQuery);
await detachFilesFromProjects(doomed);
return result;
}
/**
* Batch updates files with new signed URLs in MongoDB
* @param updates - Array of updates in the format { file_id, filepath, storageKey?, storageRegion? }
*/
async function batchUpdateFiles(
updates: Array<{
file_id: string;
filepath: string;
storageKey?: string;
storageRegion?: string;
}>,
): Promise<void> {
if (!updates || updates.length === 0) {
return;
}
const File = mongoose.models.File as Model<IMongoFile>;
const bulkOperations = updates.map((update) => ({
updateOne: {
filter: { file_id: update.file_id },
update: {
$set: {
filepath: update.filepath,
...(update.storageKey ? { storageKey: update.storageKey } : {}),
...(update.storageRegion ? { storageRegion: update.storageRegion } : {}),
},
},
},
}));
const result = await tenantSafeBulkWrite(File, bulkOperations, { timestamps: false });
logger.info(`Updated ${result.modifiedCount} files with new S3 URLs`);
}
/**
* Updates usage tracking for multiple files.
* Processes files and optional fileIds, updating their usage count in the database.
*
* @param files - Array of file objects to process
* @param fileIds - Optional array of file IDs to process
* @returns Array of updated file documents (with null results filtered out)
*/
async function updateFilesUsage(
files: Array<{ file_id: string }>,
fileIds?: string[],
options?: { user?: string; tenantId?: string | null },
): Promise<IMongoFile[]> {
const promises: Promise<IMongoFile | null>[] = [];
const seen = new Set<string>();
// Preserve the same owner scope for every deduped ID in this batch.
const user = options?.user;
const tenantId = options?.tenantId;
for (const file of files) {
const { file_id } = file;
if (seen.has(file_id)) {
continue;
}
seen.add(file_id);
promises.push(updateFileUsage({ file_id, user, tenantId }));
}
if (!fileIds) {
const results = await Promise.all(promises);
return results.filter((result): result is IMongoFile => result != null);
}
for (const file_id of fileIds) {
if (seen.has(file_id)) {
continue;
}
seen.add(file_id);
promises.push(updateFileUsage({ file_id, user, tenantId }));
}
const results = await Promise.all(promises);
return results.filter((result): result is IMongoFile => result != null);
}
/**
* Widens the upload-window TTL of owned, still-temporary files to
* `min(now + renewMs, createdAt + maxLifetimeMs)`.
*
* A renewable hold, not a release: unlike `updateFileUsage` this never
* unsets `expiresAt`, so a file that is held but never actually sent is
* still reaped once the hold lapses. Candidates are read first, then each
* doc gets a guarded write — no aggregation-pipeline update, which Amazon
* DocumentDB rejects. Four properties hold by construction, which is what
* makes the write safe to drive from a client-supplied id list:
* - capping the renewal at `createdAt + maxLifetimeMs` anchors it to an
* immutable ceiling, so repeated calls converge on a fixed deadline
* instead of walking a file's lifetime forward a window at a time;
* - renewing from `now` up to that ceiling lets a queue that is still
* draining keep its attachments alive across successive runs, while an
* abandoned queue lapses a single `renewMs` after its last touch rather
* than surviving to the ceiling;
* - the `expiresAt: { $lt: next }` write guard means a hold only ever
* widens, even against renewals landing between the read and the write;
* - `expiresAt: { $exists: true }` in the read filter and the write guard
* means a file whose TTL was already cleared by a real send stays
* permanent. Re-adding `expiresAt` there would schedule a live file for
* deletion.
*
* `createdAt` is required rather than defaulted: without the anchor there
* is no ceiling to enforce, so such a file is skipped instead of held.
*
* The owner scope is required, not optional: an unscoped call would hold
* every user's matching file. A missing owner is a no-op, not a wide
* update.
*
* @param fileIds - File IDs to hold
* @param hold - `renewMs` granted from now, capped at `maxLifetimeMs` from upload
* @param owner - Owner scope; mismatches leave the TTL unchanged
* @returns Number of files whose hold was widened
*/
async function extendFilesTTL(
fileIds: string[],
hold: { renewMs: number; maxLifetimeMs: number },
owner: { user: string; tenantId?: string | null },
): Promise<number> {
const renewMs = hold?.renewMs;
const maxLifetimeMs = hold?.maxLifetimeMs;
if (fileIds.length === 0 || !owner?.user || !(renewMs > 0) || !(maxLifetimeMs > 0)) {
return 0;
}
const File = mongoose.models.File as Model<IMongoFile>;
const filter = withOwnerScope(
{
file_id: { $in: [...new Set(fileIds)] },
expiresAt: { $exists: true },
createdAt: { $exists: true },
},
{ userId: owner.user, tenantId: owner.tenantId },
);
const renewUntil = Date.now() + renewMs;
const candidates = await File.find(filter)
.select({ _id: 1, expiresAt: 1, createdAt: 1 })
.lean<Pick<IMongoFile, '_id' | 'expiresAt' | 'createdAt'>[]>();
const holdOps = candidates.flatMap((file) => {
if (!file.createdAt || !file.expiresAt) {
return [];
}
const next = new Date(Math.min(renewUntil, file.createdAt.getTime() + maxLifetimeMs));
if (file.expiresAt.getTime() >= next.getTime()) {
return [];
}
return [
{
updateOne: {
filter: { _id: file._id, expiresAt: { $exists: true, $lt: next } },
update: { $set: { expiresAt: next } },
},
},
];
});
if (holdOps.length === 0) {
return 0;
}
/** `timestamps: false`: a hold is TTL bookkeeping, not a content write.
* Bumping `updatedAt` would also make every re-touch count as a
* modification, hiding whether the deadline actually moved. */
const result = await tenantSafeBulkWrite(File, holdOps, { timestamps: false });
return result.modifiedCount ?? 0;
}
/**
* Mark stale `status: 'pending'` file records as `'failed'` with
* `previewError: 'orphaned'`. Recovers from the one case the
* in-process deferred-preview render can't handle on its own: a
* backend restart mid-render loses the in-memory promise, leaving
* the record stuck pending forever.
*
* Cheap to run on boot — the `status` field is indexed and the typical
* cutoff (5 min) bounds the candidate set to whatever was in flight at
* the prior shutdown. The 60s render timeout means anything older
* than a few minutes that's still pending is definitively orphaned.
*
* @param maxAgeMs - Cutoff in milliseconds; records whose `updatedAt`
* is older than `now - maxAgeMs` are marked failed. Defaults to 5
* minutes (well above the 60s render ceiling).
* @returns Number of records updated.
*/
async function sweepOrphanedPreviews(maxAgeMs: number = 5 * 60 * 1000): Promise<number> {
const File = mongoose.models.File as Model<IMongoFile>;
const cutoff = new Date(Date.now() - maxAgeMs);
const result = await File.updateMany(
{ status: 'pending', updatedAt: { $lt: cutoff } },
{ $set: { status: 'failed', previewError: 'orphaned' } },
);
if (result.modifiedCount > 0) {
logger.info(
`[sweepOrphanedPreviews] Marked ${result.modifiedCount} stale 'pending' files as 'failed' (cutoff: ${cutoff.toISOString()})`,
);
}
return result.modifiedCount ?? 0;
}
return {
getRunFileCandidates,
claimRunArtifactFile,
publishRunArtifactFile,
findRunArtifactFile,
listRunArtifacts,
findFileById,
getFiles,
getProjectFiles,
getAvailableProjectFiles,
getExpiredFiles,
incrementFileDeletionAttempts,
deferExpiredFile,
getToolFilesByIds,
getCodeGeneratedFiles,
getUserCodeFiles,
getDeferredProvisionFiles,
claimCodeFile,
commitCodeFile,
createFile,
updateFile,
updateFileCodeEnvRef,
clearFileCodeEnvRefs,
addFileEmbeddedEntity,
recordFileWorkspaceImport,
clearFileEmbeddings,
isFileStorageClean,
adoptLegacySearchEmbeddings,
updateFileUsage,
deleteFile,
deleteFiles,
deleteFileByFilter,
batchUpdateFiles,
updateFilesUsage,
extendFilesTTL,
sweepOrphanedPreviews,
};
}
export type FileMethods = ReturnType<typeof createFileMethods>;