Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🧹 chore: Migrate Legacy Duplicate Code Files Blocking Dedupe Index (#14593)

Danny Avila2d606a9783f0
diff --git a/config/__tests__/migrate-code-file-duplicates.spec.js b/config/__tests__/migrate-code-file-duplicates.spec.js
new file mode 100644
index 0000000000..def867d6d1
--- /dev/null
+++ b/config/__tests__/migrate-code-file-duplicates.spec.js
@@ -0,0 +1,303 @@
+const mongoose = require('mongoose');
+const { v4: uuidv4 } = require('uuid');
+const { FileContext } = require('librechat-data-provider');
+const { logger } = require('@librechat/data-schemas');
+const { MongoMemoryServer } = require('mongodb-memory-server');
+
+// Mock the config/connect module to prevent connection attempts during tests
+jest.mock('../connect', () => jest.fn().mockResolvedValue(true));
+
+// Disable console for tests
+logger.silent = true;
+
+describe('Code File Duplicate Migration Script', () => {
+ let mongoServer;
+ let File;
+ let migrateCodeFileDuplicates;
+
+ /** The unique partial index this migration exists to unblock. */
+ const INDEX_KEYS = { filename: 1, conversationId: 1, context: 1, tenantId: 1 };
+ const INDEX_NAME = 'filename_1_conversationId_1_context_1_tenantId_1';
+ const INDEX_OPTIONS = {
+ unique: true,
+ partialFilterExpression: { context: FileContext.execute_code },
+ };
+
+ /**
+ * Reproduces the state this migration is written for: legacy duplicates
+ * present and the unique index absent because its build failed. Mongoose
+ * builds schema indexes in the background at startup, so without dropping it
+ * here the fixtures would race an index the affected deployments don't have.
+ */
+ async function dropUniqueIndex() {
+ await File.init().catch(() => {
+ /* the background build may itself fail — that IS the scenario */
+ });
+ await File.collection.dropIndex(INDEX_NAME).catch(() => {
+ /* already absent */
+ });
+ }
+
+ async function createCodeFile({ filename, conversationId, createdAt, context }) {
+ return File.create({
+ user: new mongoose.Types.ObjectId(),
+ file_id: uuidv4(),
+ filename,
+ filepath: `/images/user/${uuidv4()}.png`,
+ object: 'file',
+ type: 'image/png',
+ bytes: 1024,
+ conversationId,
+ context: context ?? FileContext.execute_code,
+ createdAt,
+ updatedAt: createdAt,
+ });
+ }
+
+ const namesFor = async (conversationId) => {
+ const files = await File.find({ conversationId }).lean();
+ return files.map((file) => file.filename).sort();
+ };
+
+ beforeAll(async () => {
+ mongoServer = await MongoMemoryServer.create();
+ await mongoose.connect(mongoServer.getUri());
+
+ const dbModels = require('~/db/models');
+ File = dbModels.File;
+
+ ({ migrateCodeFileDuplicates } = require('../migrate-code-file-duplicates'));
+
+ await dropUniqueIndex();
+ });
+
+ afterAll(async () => {
+ await mongoose.disconnect();
+ await mongoServer.stop();
+ });
+
+ afterEach(async () => {
+ await File.deleteMany({});
+ await File.collection.dropIndex(INDEX_NAME).catch(() => {
+ /* the test never built it */
+ });
+ });
+
+ it('renames older duplicates and leaves the newest record canonical', async () => {
+ const conversationId = uuidv4();
+ await createCodeFile({
+ filename: 'bar_chart.png',
+ conversationId,
+ createdAt: new Date('2025-03-29T03:54:53Z'),
+ });
+ await createCodeFile({
+ filename: 'bar_chart.png',
+ conversationId,
+ createdAt: new Date('2025-03-29T03:56:32Z'),
+ });
+
+ const result = await migrateCodeFileDuplicates({ dryRun: false });
+
+ expect(result.duplicateGroups).toBe(1);
+ expect(result.filesRenamed).toBe(1);
+ /* Newest keeps the canonical name — the claim path's "latest write wins". */
+ expect(await namesFor(conversationId)).toEqual(['bar_chart (1).png', 'bar_chart.png']);
+ const newest = await File.findOne({ filename: 'bar_chart.png' }).lean();
+ expect(newest.createdAt).toEqual(new Date('2025-03-29T03:56:32Z'));
+ });
+
+ it('never deletes: every original record survives the rename', async () => {
+ const conversationId = uuidv4();
+ const older = await createCodeFile({
+ filename: 'plot.png',
+ conversationId,
+ createdAt: new Date('2025-01-01T00:00:00Z'),
+ });
+ const newer = await createCodeFile({
+ filename: 'plot.png',
+ conversationId,
+ createdAt: new Date('2025-01-01T00:05:00Z'),
+ });
+
+ await migrateCodeFileDuplicates({ dryRun: false });
+
+ /* Both file_ids still resolve — a deleted record would strip a real
+ * artifact out of the message attachment that references it. */
+ expect(await File.countDocuments({})).toBe(2);
+ const kept = await File.findOne({ file_id: older.file_id }).lean();
+ expect(kept).not.toBeNull();
+ expect(kept.filepath).toBe(older.filepath);
+ expect((await File.findOne({ file_id: newer.file_id }).lean()).filename).toBe('plot.png');
+ });
+
+ it('unblocks the unique partial index that could not build before', async () => {
+ const conversationId = uuidv4();
+ await createCodeFile({
+ filename: 'report.png',
+ conversationId,
+ createdAt: new Date('2025-02-01T00:00:00Z'),
+ });
+ await createCodeFile({
+ filename: 'report.png',
+ conversationId,
+ createdAt: new Date('2025-02-01T00:01:00Z'),
+ });
+
+ /* Precondition: the duplicates genuinely block the build (E11000). */
+ await expect(File.collection.createIndex(INDEX_KEYS, INDEX_OPTIONS)).rejects.toThrow();
+
+ const result = await migrateCodeFileDuplicates({ dryRun: false });
+
+ expect(result.indexBuilt).toBe(true);
+ const indexes = await File.collection.indexes();
+ expect(
+ indexes.some((index) => index.name === 'filename_1_conversationId_1_context_1_tenantId_1'),
+ ).toBe(true);
+ });
+
+ it('reports without writing in dry-run mode', async () => {
+ const conversationId = uuidv4();
+ await createCodeFile({
+ filename: 'chart.png',
+ conversationId,
+ createdAt: new Date('2025-04-01T00:00:00Z'),
+ });
+ await createCodeFile({
+ filename: 'chart.png',
+ conversationId,
+ createdAt: new Date('2025-04-01T00:02:00Z'),
+ });
+
+ const result = await migrateCodeFileDuplicates({ dryRun: true });
+
+ expect(result.filesRenamed).toBe(1);
+ expect(result.indexBuilt).toBe(false);
+ expect(await namesFor(conversationId)).toEqual(['chart.png', 'chart.png']);
+ });
+
+ it('skips names already taken in the conversation', async () => {
+ const conversationId = uuidv4();
+ await createCodeFile({
+ filename: 'out.png',
+ conversationId,
+ createdAt: new Date('2025-05-01T00:00:00Z'),
+ });
+ await createCodeFile({
+ filename: 'out.png',
+ conversationId,
+ createdAt: new Date('2025-05-01T00:01:00Z'),
+ });
+ /* An unrelated record already occupies the first replacement name. */
+ await createCodeFile({
+ filename: 'out (1).png',
+ conversationId,
+ createdAt: new Date('2025-05-01T00:03:00Z'),
+ });
+
+ await migrateCodeFileDuplicates({ dryRun: false });
+
+ expect(await namesFor(conversationId)).toEqual(['out (1).png', 'out (2).png', 'out.png']);
+ });
+
+ it('resolves three copies into distinct names in one pass', async () => {
+ const conversationId = uuidv4();
+ for (const minute of [0, 1, 2]) {
+ await createCodeFile({
+ filename: 'fig.png',
+ conversationId,
+ createdAt: new Date(`2025-06-01T00:0${minute}:00Z`),
+ });
+ }
+
+ const result = await migrateCodeFileDuplicates({ dryRun: false });
+
+ expect(result.filesRenamed).toBe(2);
+ expect(await namesFor(conversationId)).toEqual(['fig (1).png', 'fig (2).png', 'fig.png']);
+ expect(result.indexBuilt).toBe(true);
+ });
+
+ it('leaves same-named files in DIFFERENT conversations alone', async () => {
+ const first = uuidv4();
+ const second = uuidv4();
+ await createCodeFile({
+ filename: 'shared.png',
+ conversationId: first,
+ createdAt: new Date('2025-07-01T00:00:00Z'),
+ });
+ await createCodeFile({
+ filename: 'shared.png',
+ conversationId: second,
+ createdAt: new Date('2025-07-01T00:01:00Z'),
+ });
+
+ const result = await migrateCodeFileDuplicates({ dryRun: false });
+
+ expect(result.duplicateGroups).toBe(0);
+ expect(result.filesRenamed).toBe(0);
+ expect(await namesFor(first)).toEqual(['shared.png']);
+ expect(await namesFor(second)).toEqual(['shared.png']);
+ });
+
+ it('ignores duplicates outside the execute_code context', async () => {
+ const conversationId = uuidv4();
+ await createCodeFile({
+ filename: 'upload.png',
+ conversationId,
+ createdAt: new Date('2025-08-01T00:00:00Z'),
+ context: FileContext.message_attachment,
+ });
+ await createCodeFile({
+ filename: 'upload.png',
+ conversationId,
+ createdAt: new Date('2025-08-01T00:01:00Z'),
+ context: FileContext.message_attachment,
+ });
+
+ const result = await migrateCodeFileDuplicates({ dryRun: false });
+
+ /* The index is partial — only code outputs must be unique. */
+ expect(result.duplicateGroups).toBe(0);
+ expect(await namesFor(conversationId)).toEqual(['upload.png', 'upload.png']);
+ });
+
+ it('is safe to re-run once each conversation is unique', async () => {
+ const conversationId = uuidv4();
+ await createCodeFile({
+ filename: 'idempotent.png',
+ conversationId,
+ createdAt: new Date('2025-09-01T00:00:00Z'),
+ });
+ await createCodeFile({
+ filename: 'idempotent.png',
+ conversationId,
+ createdAt: new Date('2025-09-01T00:01:00Z'),
+ });
+
+ await migrateCodeFileDuplicates({ dryRun: false });
+ const afterFirst = await namesFor(conversationId);
+
+ const second = await migrateCodeFileDuplicates({ dryRun: false });
+
+ expect(second.duplicateGroups).toBe(0);
+ expect(second.filesRenamed).toBe(0);
+ expect(await namesFor(conversationId)).toEqual(afterFirst);
+ });
+
+ it('handles filenames without an extension', async () => {
+ const conversationId = uuidv4();
+ await createCodeFile({
+ filename: 'Makefile',
+ conversationId,
+ createdAt: new Date('2025-10-01T00:00:00Z'),
+ });
+ await createCodeFile({
+ filename: 'Makefile',
+ conversationId,
+ createdAt: new Date('2025-10-01T00:01:00Z'),
+ });
+
+ await migrateCodeFileDuplicates({ dryRun: false });
+
+ expect(await namesFor(conversationId)).toEqual(['Makefile', 'Makefile (1)']);
+ });
+});
diff --git a/config/migrate-code-file-duplicates.js b/config/migrate-code-file-duplicates.js
new file mode 100644
index 0000000000..4dd411e0b1
--- /dev/null
+++ b/config/migrate-code-file-duplicates.js
@@ -0,0 +1,236 @@
+const path = require('path');
+const { FileContext } = require('librechat-data-provider');
+const { logger, runAsSystem } = require('@librechat/data-schemas');
+
+require('module-alias')({ base: path.resolve(__dirname, '..', 'api') });
+const connect = require('./connect');
+
+const { File } = require('~/db/models');
+
+/**
+ * Cap on the number of per-group entries retained in `results.details`. Larger
+ * runs still rename every affected record and still report accurate aggregate
+ * counts — we just stop accumulating sample data past this threshold to keep
+ * memory bounded on deployments with thousands of legacy duplicates.
+ */
+const DETAIL_SAMPLE_LIMIT = 50;
+
+/** Mirrors the unique partial index declared on the file schema. */
+const INDEX_KEYS = { filename: 1, conversationId: 1, context: 1, tenantId: 1 };
+const INDEX_OPTIONS = {
+ unique: true,
+ partialFilterExpression: { context: FileContext.execute_code },
+};
+
+/** `report.png` -> `report (2).png`; a name without an extension keeps its shape. */
+function suffixFilename(filename, n) {
+ const extension = path.extname(filename);
+ const base = extension ? filename.slice(0, -extension.length) : filename;
+ return `${base} (${n})${extension}`;
+}
+
+/**
+ * Picks a name that is free within the group's uniqueness scope. `taken` holds
+ * both the names already in the database and the ones handed out earlier in
+ * this run, so a group with several duplicates can't rename two records onto
+ * the same replacement.
+ */
+function nextAvailableName(filename, taken) {
+ for (let n = 1; ; n++) {
+ const candidate = suffixFilename(filename, n);
+ if (!taken.has(candidate)) {
+ taken.add(candidate);
+ return candidate;
+ }
+ }
+}
+
+/**
+ * Normalizes code-execution output files that share a filename within one
+ * conversation, so the unique partial index on
+ * `(filename, conversationId, context, tenantId)` can finish building.
+ *
+ * That index arrived with atomic file claiming: a regenerated output now
+ * converges on ONE record with a cache-busted filepath. Records written before
+ * that change instead inserted a second document per regeneration, so any
+ * deployment that re-ran a code cell producing the same filename carries
+ * duplicates the index cannot span. Mongo then aborts the build with E11000 and
+ * the constraint is silently absent — the claim path keeps working, but without
+ * its database-level guarantee against concurrent inserts.
+ *
+ * Renames rather than deletes: every duplicate is a distinct stored object,
+ * usually still referenced by a message attachment, so removing one would
+ * strip a real artifact out of a user's history. The newest record keeps the
+ * canonical name (matching the "latest write wins" behavior of the claim path);
+ * older ones gain a ` (n)` suffix. Attachments carry their own filename copy,
+ * so rendered history is untouched.
+ *
+ * Safe to re-run — once each scope is unique, nothing is written.
+ *
+ * @param {{ dryRun?: boolean, batchSize?: number }} [options]
+ */
+async function migrateCodeFileDuplicates({ dryRun = true, batchSize = 100 } = {}) {
+ await connect();
+
+ logger.info('Starting Code File Duplicate Migration', { dryRun, batchSize });
+
+ /*
+ * Scan and heal across every tenant. Without this wrapper the tenant
+ * isolation plugin either scopes queries to a (non-existent) tenant or
+ * throws under TENANT_ISOLATION_STRICT=true, making the script unusable as
+ * the intended remediation path.
+ */
+ return runAsSystem(async () => {
+ const results = {
+ dryRun,
+ scannedFiles: 0,
+ duplicateGroups: 0,
+ filesRenamed: 0,
+ indexBuilt: false,
+ errors: 0,
+ details: [],
+ };
+
+ results.scannedFiles = await File.countDocuments({ context: FileContext.execute_code });
+ logger.info(`Scanning ${results.scannedFiles} code-execution file(s) for duplicates`);
+
+ const groups = await File.aggregate([
+ { $match: { context: FileContext.execute_code } },
+ {
+ $group: {
+ _id: {
+ filename: '$filename',
+ conversationId: '$conversationId',
+ tenantId: '$tenantId',
+ },
+ count: { $sum: 1 },
+ files: { $push: { _id: '$_id', file_id: '$file_id', createdAt: '$createdAt' } },
+ },
+ },
+ { $match: { count: { $gt: 1 } } },
+ ]).option({ batchSize });
+
+ results.duplicateGroups = groups.length;
+
+ for (const group of groups) {
+ try {
+ /* Newest first: it keeps the canonical name, older copies get suffixed. */
+ const ordered = [...group.files].sort(
+ (a, b) => new Date(b.createdAt ?? 0) - new Date(a.createdAt ?? 0),
+ );
+ const [, ...stale] = ordered;
+
+ /* Reserve every name already used in this uniqueness scope, so a
+ * suffixed replacement can't collide with an unrelated record that
+ * happens to be called `report (1).png` already. */
+ const scopeNames = await File.find(
+ {
+ context: FileContext.execute_code,
+ conversationId: group._id.conversationId ?? null,
+ tenantId: group._id.tenantId ?? null,
+ },
+ { filename: 1, _id: 0 },
+ ).lean();
+ const taken = new Set(scopeNames.map((file) => file.filename));
+
+ const renames = stale.map((file) => ({
+ file_id: file.file_id,
+ _id: file._id,
+ from: group._id.filename,
+ to: nextAvailableName(group._id.filename, taken),
+ }));
+
+ if (!dryRun) {
+ for (const rename of renames) {
+ await File.updateOne({ _id: rename._id }, { $set: { filename: rename.to } });
+ }
+ }
+
+ results.filesRenamed += renames.length;
+ if (results.details.length < DETAIL_SAMPLE_LIMIT) {
+ results.details.push({
+ filename: group._id.filename,
+ conversationId: group._id.conversationId,
+ count: group.count,
+ renames: renames.map(({ file_id, to }) => ({ file_id, to })),
+ });
+ }
+ } catch (error) {
+ results.errors++;
+ logger.error(
+ `Failed to normalize duplicates for "${group._id.filename}" in conversation ${group._id.conversationId}: ${error.message}`,
+ );
+ }
+ }
+
+ /*
+ * Build the index here rather than waiting for the next boot: the operator
+ * ran this to fix a failing build, so they should learn immediately whether
+ * it now succeeds. Targeted `createIndex` (not `syncIndexes`, which would
+ * drop indexes absent from the schema).
+ */
+ if (!dryRun && results.errors === 0) {
+ try {
+ await File.collection.createIndex(INDEX_KEYS, { ...INDEX_OPTIONS, background: true });
+ results.indexBuilt = true;
+ } catch (error) {
+ results.errors++;
+ logger.error(
+ `Duplicates normalized but the unique index still failed to build: ${error.message}`,
+ );
+ }
+ }
+
+ logger.info('Code File Duplicate Migration completed', {
+ dryRun,
+ scannedFiles: results.scannedFiles,
+ duplicateGroups: results.duplicateGroups,
+ filesRenamed: results.filesRenamed,
+ indexBuilt: results.indexBuilt,
+ errors: results.errors,
+ });
+
+ return results;
+ });
+}
+
+if (require.main === module) {
+ const dryRun = process.argv.includes('--dry-run');
+ const batchSize =
+ parseInt(process.argv.find((arg) => arg.startsWith('--batch-size='))?.split('=')[1]) || 100;
+
+ migrateCodeFileDuplicates({ dryRun, batchSize })
+ .then((result) => {
+ console.log(`\n=== ${dryRun ? 'DRY RUN ' : ''}RESULTS ===`);
+ console.log(`Code files scanned: ${result.scannedFiles}`);
+ console.log(`Duplicate groups: ${result.duplicateGroups}`);
+ console.log(`Files ${dryRun ? 'to rename' : 'renamed'}: ${result.filesRenamed}`);
+ if (!dryRun && result.duplicateGroups > 0) {
+ console.log(`Unique index built: ${result.indexBuilt ? 'yes' : 'no'}`);
+ }
+ if (result.errors > 0) {
+ console.log(`Errors: ${result.errors}`);
+ }
+ if (result.details.length > 0) {
+ console.log('\nAffected files:');
+ result.details.forEach((d, i) => {
+ console.log(
+ ` ${i + 1}. "${d.filename}" in ${d.conversationId} — ${d.count} copies, ${d.renames.length} renamed`,
+ );
+ d.renames.forEach((r) => console.log(` ${r.file_id} -> "${r.to}"`));
+ });
+ if (result.duplicateGroups > result.details.length) {
+ console.log(
+ ` ... and ${result.duplicateGroups - result.details.length} more (sample capped at ${DETAIL_SAMPLE_LIMIT})`,
+ );
+ }
+ }
+ process.exit(0);
+ })
+ .catch((error) => {
+ console.error('Code file duplicate migration failed:', error);
+ process.exit(1);
+ });
+}
+
+module.exports = { migrateCodeFileDuplicates };
diff --git a/package.json b/package.json
index 8b836380e7..5fb6c358a3 100644
--- a/package.json
+++ b/package.json
@@ -111,7 +111,10 @@
"migrate:shared-link-permissions:batch": "node config/migrate-shared-link-permissions.js --batch-size=50",
"migrate:orphaned-agent-files:dry-run": "node config/migrate-orphaned-agent-files.js --dry-run",
"migrate:orphaned-agent-files": "node config/migrate-orphaned-agent-files.js",
- "migrate:orphaned-agent-files:batch": "node config/migrate-orphaned-agent-files.js --batch-size=50"
+ "migrate:orphaned-agent-files:batch": "node config/migrate-orphaned-agent-files.js --batch-size=50",
+ "migrate:code-file-duplicates:dry-run": "node config/migrate-code-file-duplicates.js --dry-run",
+ "migrate:code-file-duplicates": "node config/migrate-code-file-duplicates.js",
+ "migrate:code-file-duplicates:batch": "node config/migrate-code-file-duplicates.js --batch-size=50"
},
"repository": {
"type": "git",