Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🧹 chore: Migrate Legacy Duplicate Code Files Blocking Dedupe Index (#14593)
Danny Avila2d606a9783f0
diff --git a/config/__tests__/migrate-code-file-duplicates.spec.js b/config/__tests__/migrate-code-file-duplicates.spec.jsnew file mode 100644index 0000000000..def867d6d1--- /dev/null+++ b/config/__tests__/migrate-code-file-duplicates.spec.js@@ -0,0 +1,303 @@+const mongoose = require('mongoose');+const { v4: uuidv4 } = require('uuid');+const { FileContext } = require('librechat-data-provider');+const { logger } = require('@librechat/data-schemas');+const { MongoMemoryServer } = require('mongodb-memory-server');++// Mock the config/connect module to prevent connection attempts during tests+jest.mock('../connect', () => jest.fn().mockResolvedValue(true));++// Disable console for tests+logger.silent = true;++describe('Code File Duplicate Migration Script', () => {+ let mongoServer;+ let File;+ let migrateCodeFileDuplicates;++ /** The unique partial index this migration exists to unblock. */+ const INDEX_KEYS = { filename: 1, conversationId: 1, context: 1, tenantId: 1 };+ const INDEX_NAME = 'filename_1_conversationId_1_context_1_tenantId_1';+ const INDEX_OPTIONS = {+ unique: true,+ partialFilterExpression: { context: FileContext.execute_code },+ };++ /**+ * Reproduces the state this migration is written for: legacy duplicates+ * present and the unique index absent because its build failed. Mongoose+ * builds schema indexes in the background at startup, so without dropping it+ * here the fixtures would race an index the affected deployments don't have.+ */+ async function dropUniqueIndex() {+ await File.init().catch(() => {+ /* the background build may itself fail — that IS the scenario */+ });+ await File.collection.dropIndex(INDEX_NAME).catch(() => {+ /* already absent */+ });+ }++ async function createCodeFile({ filename, conversationId, createdAt, context }) {+ return File.create({+ user: new mongoose.Types.ObjectId(),+ file_id: uuidv4(),+ filename,+ filepath: `/images/user/${uuidv4()}.png`,+ object: 'file',+ type: 'image/png',+ bytes: 1024,+ conversationId,+ context: context ?? FileContext.execute_code,+ createdAt,+ updatedAt: createdAt,+ });+ }++ const namesFor = async (conversationId) => {+ const files = await File.find({ conversationId }).lean();+ return files.map((file) => file.filename).sort();+ };++ beforeAll(async () => {+ mongoServer = await MongoMemoryServer.create();+ await mongoose.connect(mongoServer.getUri());++ const dbModels = require('~/db/models');+ File = dbModels.File;++ ({ migrateCodeFileDuplicates } = require('../migrate-code-file-duplicates'));++ await dropUniqueIndex();+ });++ afterAll(async () => {+ await mongoose.disconnect();+ await mongoServer.stop();+ });++ afterEach(async () => {+ await File.deleteMany({});+ await File.collection.dropIndex(INDEX_NAME).catch(() => {+ /* the test never built it */+ });+ });++ it('renames older duplicates and leaves the newest record canonical', async () => {+ const conversationId = uuidv4();+ await createCodeFile({+ filename: 'bar_chart.png',+ conversationId,+ createdAt: new Date('2025-03-29T03:54:53Z'),+ });+ await createCodeFile({+ filename: 'bar_chart.png',+ conversationId,+ createdAt: new Date('2025-03-29T03:56:32Z'),+ });++ const result = await migrateCodeFileDuplicates({ dryRun: false });++ expect(result.duplicateGroups).toBe(1);+ expect(result.filesRenamed).toBe(1);+ /* Newest keeps the canonical name — the claim path's "latest write wins". */+ expect(await namesFor(conversationId)).toEqual(['bar_chart (1).png', 'bar_chart.png']);+ const newest = await File.findOne({ filename: 'bar_chart.png' }).lean();+ expect(newest.createdAt).toEqual(new Date('2025-03-29T03:56:32Z'));+ });++ it('never deletes: every original record survives the rename', async () => {+ const conversationId = uuidv4();+ const older = await createCodeFile({+ filename: 'plot.png',+ conversationId,+ createdAt: new Date('2025-01-01T00:00:00Z'),+ });+ const newer = await createCodeFile({+ filename: 'plot.png',+ conversationId,+ createdAt: new Date('2025-01-01T00:05:00Z'),+ });++ await migrateCodeFileDuplicates({ dryRun: false });++ /* Both file_ids still resolve — a deleted record would strip a real+ * artifact out of the message attachment that references it. */+ expect(await File.countDocuments({})).toBe(2);+ const kept = await File.findOne({ file_id: older.file_id }).lean();+ expect(kept).not.toBeNull();+ expect(kept.filepath).toBe(older.filepath);+ expect((await File.findOne({ file_id: newer.file_id }).lean()).filename).toBe('plot.png');+ });++ it('unblocks the unique partial index that could not build before', async () => {+ const conversationId = uuidv4();+ await createCodeFile({+ filename: 'report.png',+ conversationId,+ createdAt: new Date('2025-02-01T00:00:00Z'),+ });+ await createCodeFile({+ filename: 'report.png',+ conversationId,+ createdAt: new Date('2025-02-01T00:01:00Z'),+ });++ /* Precondition: the duplicates genuinely block the build (E11000). */+ await expect(File.collection.createIndex(INDEX_KEYS, INDEX_OPTIONS)).rejects.toThrow();++ const result = await migrateCodeFileDuplicates({ dryRun: false });++ expect(result.indexBuilt).toBe(true);+ const indexes = await File.collection.indexes();+ expect(+ indexes.some((index) => index.name === 'filename_1_conversationId_1_context_1_tenantId_1'),+ ).toBe(true);+ });++ it('reports without writing in dry-run mode', async () => {+ const conversationId = uuidv4();+ await createCodeFile({+ filename: 'chart.png',+ conversationId,+ createdAt: new Date('2025-04-01T00:00:00Z'),+ });+ await createCodeFile({+ filename: 'chart.png',+ conversationId,+ createdAt: new Date('2025-04-01T00:02:00Z'),+ });++ const result = await migrateCodeFileDuplicates({ dryRun: true });++ expect(result.filesRenamed).toBe(1);+ expect(result.indexBuilt).toBe(false);+ expect(await namesFor(conversationId)).toEqual(['chart.png', 'chart.png']);+ });++ it('skips names already taken in the conversation', async () => {+ const conversationId = uuidv4();+ await createCodeFile({+ filename: 'out.png',+ conversationId,+ createdAt: new Date('2025-05-01T00:00:00Z'),+ });+ await createCodeFile({+ filename: 'out.png',+ conversationId,+ createdAt: new Date('2025-05-01T00:01:00Z'),+ });+ /* An unrelated record already occupies the first replacement name. */+ await createCodeFile({+ filename: 'out (1).png',+ conversationId,+ createdAt: new Date('2025-05-01T00:03:00Z'),+ });++ await migrateCodeFileDuplicates({ dryRun: false });++ expect(await namesFor(conversationId)).toEqual(['out (1).png', 'out (2).png', 'out.png']);+ });++ it('resolves three copies into distinct names in one pass', async () => {+ const conversationId = uuidv4();+ for (const minute of [0, 1, 2]) {+ await createCodeFile({+ filename: 'fig.png',+ conversationId,+ createdAt: new Date(`2025-06-01T00:0${minute}:00Z`),+ });+ }++ const result = await migrateCodeFileDuplicates({ dryRun: false });++ expect(result.filesRenamed).toBe(2);+ expect(await namesFor(conversationId)).toEqual(['fig (1).png', 'fig (2).png', 'fig.png']);+ expect(result.indexBuilt).toBe(true);+ });++ it('leaves same-named files in DIFFERENT conversations alone', async () => {+ const first = uuidv4();+ const second = uuidv4();+ await createCodeFile({+ filename: 'shared.png',+ conversationId: first,+ createdAt: new Date('2025-07-01T00:00:00Z'),+ });+ await createCodeFile({+ filename: 'shared.png',+ conversationId: second,+ createdAt: new Date('2025-07-01T00:01:00Z'),+ });++ const result = await migrateCodeFileDuplicates({ dryRun: false });++ expect(result.duplicateGroups).toBe(0);+ expect(result.filesRenamed).toBe(0);+ expect(await namesFor(first)).toEqual(['shared.png']);+ expect(await namesFor(second)).toEqual(['shared.png']);+ });++ it('ignores duplicates outside the execute_code context', async () => {+ const conversationId = uuidv4();+ await createCodeFile({+ filename: 'upload.png',+ conversationId,+ createdAt: new Date('2025-08-01T00:00:00Z'),+ context: FileContext.message_attachment,+ });+ await createCodeFile({+ filename: 'upload.png',+ conversationId,+ createdAt: new Date('2025-08-01T00:01:00Z'),+ context: FileContext.message_attachment,+ });++ const result = await migrateCodeFileDuplicates({ dryRun: false });++ /* The index is partial — only code outputs must be unique. */+ expect(result.duplicateGroups).toBe(0);+ expect(await namesFor(conversationId)).toEqual(['upload.png', 'upload.png']);+ });++ it('is safe to re-run once each conversation is unique', async () => {+ const conversationId = uuidv4();+ await createCodeFile({+ filename: 'idempotent.png',+ conversationId,+ createdAt: new Date('2025-09-01T00:00:00Z'),+ });+ await createCodeFile({+ filename: 'idempotent.png',+ conversationId,+ createdAt: new Date('2025-09-01T00:01:00Z'),+ });++ await migrateCodeFileDuplicates({ dryRun: false });+ const afterFirst = await namesFor(conversationId);++ const second = await migrateCodeFileDuplicates({ dryRun: false });++ expect(second.duplicateGroups).toBe(0);+ expect(second.filesRenamed).toBe(0);+ expect(await namesFor(conversationId)).toEqual(afterFirst);+ });++ it('handles filenames without an extension', async () => {+ const conversationId = uuidv4();+ await createCodeFile({+ filename: 'Makefile',+ conversationId,+ createdAt: new Date('2025-10-01T00:00:00Z'),+ });+ await createCodeFile({+ filename: 'Makefile',+ conversationId,+ createdAt: new Date('2025-10-01T00:01:00Z'),+ });++ await migrateCodeFileDuplicates({ dryRun: false });++ expect(await namesFor(conversationId)).toEqual(['Makefile', 'Makefile (1)']);+ });+});diff --git a/config/migrate-code-file-duplicates.js b/config/migrate-code-file-duplicates.jsnew file mode 100644index 0000000000..4dd411e0b1--- /dev/null+++ b/config/migrate-code-file-duplicates.js@@ -0,0 +1,236 @@+const path = require('path');+const { FileContext } = require('librechat-data-provider');+const { logger, runAsSystem } = require('@librechat/data-schemas');++require('module-alias')({ base: path.resolve(__dirname, '..', 'api') });+const connect = require('./connect');++const { File } = require('~/db/models');++/**+ * Cap on the number of per-group entries retained in `results.details`. Larger+ * runs still rename every affected record and still report accurate aggregate+ * counts — we just stop accumulating sample data past this threshold to keep+ * memory bounded on deployments with thousands of legacy duplicates.+ */+const DETAIL_SAMPLE_LIMIT = 50;++/** Mirrors the unique partial index declared on the file schema. */+const INDEX_KEYS = { filename: 1, conversationId: 1, context: 1, tenantId: 1 };+const INDEX_OPTIONS = {+ unique: true,+ partialFilterExpression: { context: FileContext.execute_code },+};++/** `report.png` -> `report (2).png`; a name without an extension keeps its shape. */+function suffixFilename(filename, n) {+ const extension = path.extname(filename);+ const base = extension ? filename.slice(0, -extension.length) : filename;+ return `${base} (${n})${extension}`;+}++/**+ * Picks a name that is free within the group's uniqueness scope. `taken` holds+ * both the names already in the database and the ones handed out earlier in+ * this run, so a group with several duplicates can't rename two records onto+ * the same replacement.+ */+function nextAvailableName(filename, taken) {+ for (let n = 1; ; n++) {+ const candidate = suffixFilename(filename, n);+ if (!taken.has(candidate)) {+ taken.add(candidate);+ return candidate;+ }+ }+}++/**+ * Normalizes code-execution output files that share a filename within one+ * conversation, so the unique partial index on+ * `(filename, conversationId, context, tenantId)` can finish building.+ *+ * That index arrived with atomic file claiming: a regenerated output now+ * converges on ONE record with a cache-busted filepath. Records written before+ * that change instead inserted a second document per regeneration, so any+ * deployment that re-ran a code cell producing the same filename carries+ * duplicates the index cannot span. Mongo then aborts the build with E11000 and+ * the constraint is silently absent — the claim path keeps working, but without+ * its database-level guarantee against concurrent inserts.+ *+ * Renames rather than deletes: every duplicate is a distinct stored object,+ * usually still referenced by a message attachment, so removing one would+ * strip a real artifact out of a user's history. The newest record keeps the+ * canonical name (matching the "latest write wins" behavior of the claim path);+ * older ones gain a ` (n)` suffix. Attachments carry their own filename copy,+ * so rendered history is untouched.+ *+ * Safe to re-run — once each scope is unique, nothing is written.+ *+ * @param {{ dryRun?: boolean, batchSize?: number }} [options]+ */+async function migrateCodeFileDuplicates({ dryRun = true, batchSize = 100 } = {}) {+ await connect();++ logger.info('Starting Code File Duplicate Migration', { dryRun, batchSize });++ /*+ * Scan and heal across every tenant. Without this wrapper the tenant+ * isolation plugin either scopes queries to a (non-existent) tenant or+ * throws under TENANT_ISOLATION_STRICT=true, making the script unusable as+ * the intended remediation path.+ */+ return runAsSystem(async () => {+ const results = {+ dryRun,+ scannedFiles: 0,+ duplicateGroups: 0,+ filesRenamed: 0,+ indexBuilt: false,+ errors: 0,+ details: [],+ };++ results.scannedFiles = await File.countDocuments({ context: FileContext.execute_code });+ logger.info(`Scanning ${results.scannedFiles} code-execution file(s) for duplicates`);++ const groups = await File.aggregate([+ { $match: { context: FileContext.execute_code } },+ {+ $group: {+ _id: {+ filename: '$filename',+ conversationId: '$conversationId',+ tenantId: '$tenantId',+ },+ count: { $sum: 1 },+ files: { $push: { _id: '$_id', file_id: '$file_id', createdAt: '$createdAt' } },+ },+ },+ { $match: { count: { $gt: 1 } } },+ ]).option({ batchSize });++ results.duplicateGroups = groups.length;++ for (const group of groups) {+ try {+ /* Newest first: it keeps the canonical name, older copies get suffixed. */+ const ordered = [...group.files].sort(+ (a, b) => new Date(b.createdAt ?? 0) - new Date(a.createdAt ?? 0),+ );+ const [, ...stale] = ordered;++ /* Reserve every name already used in this uniqueness scope, so a+ * suffixed replacement can't collide with an unrelated record that+ * happens to be called `report (1).png` already. */+ const scopeNames = await File.find(+ {+ context: FileContext.execute_code,+ conversationId: group._id.conversationId ?? null,+ tenantId: group._id.tenantId ?? null,+ },+ { filename: 1, _id: 0 },+ ).lean();+ const taken = new Set(scopeNames.map((file) => file.filename));++ const renames = stale.map((file) => ({+ file_id: file.file_id,+ _id: file._id,+ from: group._id.filename,+ to: nextAvailableName(group._id.filename, taken),+ }));++ if (!dryRun) {+ for (const rename of renames) {+ await File.updateOne({ _id: rename._id }, { $set: { filename: rename.to } });+ }+ }++ results.filesRenamed += renames.length;+ if (results.details.length < DETAIL_SAMPLE_LIMIT) {+ results.details.push({+ filename: group._id.filename,+ conversationId: group._id.conversationId,+ count: group.count,+ renames: renames.map(({ file_id, to }) => ({ file_id, to })),+ });+ }+ } catch (error) {+ results.errors++;+ logger.error(+ `Failed to normalize duplicates for "${group._id.filename}" in conversation ${group._id.conversationId}: ${error.message}`,+ );+ }+ }++ /*+ * Build the index here rather than waiting for the next boot: the operator+ * ran this to fix a failing build, so they should learn immediately whether+ * it now succeeds. Targeted `createIndex` (not `syncIndexes`, which would+ * drop indexes absent from the schema).+ */+ if (!dryRun && results.errors === 0) {+ try {+ await File.collection.createIndex(INDEX_KEYS, { ...INDEX_OPTIONS, background: true });+ results.indexBuilt = true;+ } catch (error) {+ results.errors++;+ logger.error(+ `Duplicates normalized but the unique index still failed to build: ${error.message}`,+ );+ }+ }++ logger.info('Code File Duplicate Migration completed', {+ dryRun,+ scannedFiles: results.scannedFiles,+ duplicateGroups: results.duplicateGroups,+ filesRenamed: results.filesRenamed,+ indexBuilt: results.indexBuilt,+ errors: results.errors,+ });++ return results;+ });+}++if (require.main === module) {+ const dryRun = process.argv.includes('--dry-run');+ const batchSize =+ parseInt(process.argv.find((arg) => arg.startsWith('--batch-size='))?.split('=')[1]) || 100;++ migrateCodeFileDuplicates({ dryRun, batchSize })+ .then((result) => {+ console.log(`\n=== ${dryRun ? 'DRY RUN ' : ''}RESULTS ===`);+ console.log(`Code files scanned: ${result.scannedFiles}`);+ console.log(`Duplicate groups: ${result.duplicateGroups}`);+ console.log(`Files ${dryRun ? 'to rename' : 'renamed'}: ${result.filesRenamed}`);+ if (!dryRun && result.duplicateGroups > 0) {+ console.log(`Unique index built: ${result.indexBuilt ? 'yes' : 'no'}`);+ }+ if (result.errors > 0) {+ console.log(`Errors: ${result.errors}`);+ }+ if (result.details.length > 0) {+ console.log('\nAffected files:');+ result.details.forEach((d, i) => {+ console.log(+ ` ${i + 1}. "${d.filename}" in ${d.conversationId} — ${d.count} copies, ${d.renames.length} renamed`,+ );+ d.renames.forEach((r) => console.log(` ${r.file_id} -> "${r.to}"`));+ });+ if (result.duplicateGroups > result.details.length) {+ console.log(+ ` ... and ${result.duplicateGroups - result.details.length} more (sample capped at ${DETAIL_SAMPLE_LIMIT})`,+ );+ }+ }+ process.exit(0);+ })+ .catch((error) => {+ console.error('Code file duplicate migration failed:', error);+ process.exit(1);+ });+}++module.exports = { migrateCodeFileDuplicates };diff --git a/package.json b/package.jsonindex 8b836380e7..5fb6c358a3 100644--- a/package.json+++ b/package.json@@ -111,7 +111,10 @@"migrate:shared-link-permissions:batch": "node config/migrate-shared-link-permissions.js --batch-size=50","migrate:orphaned-agent-files:dry-run": "node config/migrate-orphaned-agent-files.js --dry-run","migrate:orphaned-agent-files": "node config/migrate-orphaned-agent-files.js",- "migrate:orphaned-agent-files:batch": "node config/migrate-orphaned-agent-files.js --batch-size=50"+ "migrate:orphaned-agent-files:batch": "node config/migrate-orphaned-agent-files.js --batch-size=50",+ "migrate:code-file-duplicates:dry-run": "node config/migrate-code-file-duplicates.js --dry-run",+ "migrate:code-file-duplicates": "node config/migrate-code-file-duplicates.js",+ "migrate:code-file-duplicates:batch": "node config/migrate-code-file-duplicates.js --batch-size=50"},"repository": {"type": "git",