Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🔄 fix: Reindex MeiliSearch Documents After Indexed-Projection Changes (#15484)
csdivad8a40e9f6752a
diff --git a/packages/data-schemas/src/models/plugins/mongoMeili.spec.ts b/packages/data-schemas/src/models/plugins/mongoMeili.spec.tsindex 4b848289df..fab10b58d2 100644--- a/packages/data-schemas/src/models/plugins/mongoMeili.spec.ts+++ b/packages/data-schemas/src/models/plugins/mongoMeili.spec.ts@@ -1,7 +1,8 @@import mongoose from 'mongoose';import { EModelEndpoint } from 'librechat-data-provider';import { MongoMemoryServer } from 'mongodb-memory-server';-import mongoMeili, { type SchemaWithMeiliMethods } from '~/models/plugins/mongoMeili';+import type { SchemaWithMeiliMethods } from '~/models/plugins/mongoMeili';+import mongoMeili, { MEILI_INDEX_SCHEMA_VERSION } from '~/models/plugins/mongoMeili';import { createConversationModel } from '~/models/convo';import { createMessageModel } from '~/models/message';@@ -13,6 +14,7 @@ interface DynamicMeiliDocument extends mongoose.Document {expiredAt?: Date | null;_meiliIndex?: boolean;_meiliIndexAttempted?: boolean;+ _meiliIndexSchemaVersion?: number;_meiliCleanupVersion?: number;}@@ -1317,6 +1319,7 @@ describe('Meilisearch Mongoose plugin', () => {title: 'Indexed',endpoint: EModelEndpoint.openAI,_meiliIndex: true,+ _meiliIndexSchemaVersion: MEILI_INDEX_SCHEMA_VERSION,expiredAt: null,});@@ -1336,6 +1339,36 @@ describe('Meilisearch Mongoose plugin', () => {expect(progress.isComplete).toBe(false);});+ test('reindexes documents from an older indexed schema version', async () => {+ const conversationModel = mongoose.models.Conversation as SchemaWithMeiliMethods;+ await conversationModel.deleteMany({});++ await conversationModel.collection.insertOne({+ conversationId: new mongoose.Types.ObjectId().toString(),+ user: new mongoose.Types.ObjectId(),+ title: 'Legacy Indexed Conversation',+ endpoint: EModelEndpoint.openAI,+ _meiliIndex: true,+ _meiliIndexSchemaVersion: MEILI_INDEX_SCHEMA_VERSION - 1,+ expiredAt: null,+ });++ const progress = await conversationModel.getSyncProgress();++ expect(progress.totalProcessed).toBe(0);+ expect(progress.pendingIndexing).toBe(1);++ await conversationModel.syncWithMeili();++ expect(+ (+ await conversationModel+ .findOne({ title: 'Legacy Indexed Conversation' })+ .select('+_meiliIndexSchemaVersion')+ )?._meiliIndexSchemaVersion,+ ).toBe(MEILI_INDEX_SCHEMA_VERSION);+ });+test('getSyncProgress excludes TTL documents from counts', async () => {const conversationModel = createConversationModel(mongoose,@@ -1349,6 +1382,7 @@ describe('Meilisearch Mongoose plugin', () => {title: 'Syncable Indexed',endpoint: EModelEndpoint.openAI,_meiliIndex: true,+ _meiliIndexSchemaVersion: MEILI_INDEX_SCHEMA_VERSION,expiredAt: null,});@@ -1399,6 +1433,7 @@ describe('Meilisearch Mongoose plugin', () => {user: new mongoose.Types.ObjectId(),isCreatedByUser: true,_meiliIndex: true,+ _meiliIndexSchemaVersion: MEILI_INDEX_SCHEMA_VERSION,expiredAt: null,});@@ -1408,6 +1443,7 @@ describe('Meilisearch Mongoose plugin', () => {user: new mongoose.Types.ObjectId(),isCreatedByUser: false,_meiliIndex: true,+ _meiliIndexSchemaVersion: MEILI_INDEX_SCHEMA_VERSION,expiredAt: null,});@@ -2158,6 +2194,7 @@ describe('Meilisearch Mongoose plugin', () => {endpoint: EModelEndpoint.openAI,expiredAt: null,_meiliIndex: true,+ _meiliIndexSchemaVersion: MEILI_INDEX_SCHEMA_VERSION,},]);@@ -2196,6 +2233,7 @@ describe('Meilisearch Mongoose plugin', () => {isCreatedByUser: true,expiredAt: null,_meiliIndex: true,+ _meiliIndexSchemaVersion: MEILI_INDEX_SCHEMA_VERSION,},{messageId: new mongoose.Types.ObjectId(),diff --git a/packages/data-schemas/src/models/plugins/mongoMeili.ts b/packages/data-schemas/src/models/plugins/mongoMeili.tsindex f1c8ce9c19..f835f28a92 100644--- a/packages/data-schemas/src/models/plugins/mongoMeili.ts+++ b/packages/data-schemas/src/models/plugins/mongoMeili.ts@@ -32,6 +32,7 @@ interface MeiliIndexable {_meiliIndex?: boolean;_meiliIndexAttempted?: boolean;_meiliIndexVersion?: string;+ _meiliIndexSchemaVersion?: number;_meiliCleanupVersion?: number;}@@ -48,6 +49,7 @@ interface _DocumentWithMeiliIndex extends Document {_meiliIndex?: boolean;_meiliIndexAttempted?: boolean;_meiliIndexVersion?: string;+ _meiliIndexSchemaVersion?: number;_meiliCleanupVersion?: number;isTemporary?: boolean;expiredAt?: Date | null;@@ -107,6 +109,9 @@ const getSyncConfig = () => ({const hasSchemaPath = (schema: Schema, path: string): boolean =>Object.prototype.hasOwnProperty.call(schema.obj, path);+/** Bump when the indexed document shape or projection changes. */+export const MEILI_INDEX_SCHEMA_VERSION = 1;+const explicitTemporaryFlagKey = 'meiliExplicitTemporaryFlag';const previouslyIndexedFlagKey = 'meiliPreviouslyIndexed';const meiliCleanupVersion = 1;@@ -304,6 +309,7 @@ const createMeiliMongooseModel = ({'+_meiliIndex','+_meiliIndexAttempted','+_meiliIndexVersion',+ '+_meiliIndexSchemaVersion','isTemporary','expiredAt','unfinished',@@ -378,7 +384,13 @@ const createMeiliMongooseModel = ({// eslint-disable-next-line no-restricted-syntax -- versioned internal bookkeeping must not re-enter document middlewareconst acknowledgement = await doc.collection.updateOne({ _id: doc._id as Types.ObjectId, _meiliIndexVersion: version },- { $set: { _meiliIndex: true, _meiliCleanupVersion: meiliCleanupVersion } },+ {+ $set: {+ _meiliIndex: true,+ _meiliCleanupVersion: meiliCleanupVersion,+ _meiliIndexSchemaVersion: MEILI_INDEX_SCHEMA_VERSION,+ },+ },);if (acknowledgement.matchedCount > 0) {return;@@ -412,36 +424,49 @@ const createMeiliMongooseModel = ({static async getSyncProgress(this: SchemaWithMeiliMethods): Promise<SyncProgress> {const indexableQuery = getIndexableQuery();const excludedIndexedQuery = getExcludedIndexedQuery();- const [totalDocuments, indexedDocuments, pendingIndexing, pendingCleanup] = await Promise.all(- [- this.countDocuments(indexableQuery),- this.countDocuments({- ...indexableQuery,- _meiliIndex: true,- }),- this.countDocuments({- ...indexableQuery,- _meiliIndex: { $ne: true },- _meiliIndexAttempted: true,- }),- excludedIndexedQuery == null- ? Promise.resolve(0)- : this.countDocuments(excludedIndexedQuery),+ const needsIndexingQuery: FilterQuery<unknown> = {+ $and: [+ indexableQuery,+ {+ $or: [+ { _meiliIndex: { $ne: true }, _meiliIndexAttempted: true },+ {+ _meiliIndex: true,+ _meiliIndexSchemaVersion: { $ne: MEILI_INDEX_SCHEMA_VERSION },+ },+ ],+ },],- );+ };+ const [totalDocuments, totalProcessed, pendingIndexing, pendingCleanup] = await Promise.all([+ this.countDocuments(indexableQuery),+ this.countDocuments({+ $and: [+ indexableQuery,+ {+ _meiliIndex: true,+ _meiliIndexSchemaVersion: MEILI_INDEX_SCHEMA_VERSION,+ },+ ],+ }),+ this.countDocuments(needsIndexingQuery),+ excludedIndexedQuery == null+ ? Promise.resolve(0)+ : this.countDocuments(excludedIndexedQuery),+ ]);return {- totalProcessed: indexedDocuments,+ totalProcessed,totalDocuments,pendingIndexing,pendingCleanup,- isComplete: indexedDocuments === totalDocuments && pendingCleanup === 0,+ isComplete: totalProcessed === totalDocuments && pendingCleanup === 0,};}/*** Synchronizes data between the MongoDB collection and the MeiliSearch index by- * incrementally indexing only non-temporary documents where `_meiliIndex` is not `true`.+ * incrementally indexing only non-temporary documents that are unindexed or stale.* */static async syncWithMeili(this: SchemaWithMeiliMethods): Promise<void> {const startTime = Date.now();@@ -474,8 +499,18 @@ const createMeiliMongooseModel = ({while (hasMore) {const indexableQuery = getIndexableQuery();const query: FilterQuery<unknown> = {- ...indexableQuery,- _meiliIndex: { $ne: true },+ $and: [+ indexableQuery,+ {+ $or: [+ { _meiliIndex: { $ne: true } },+ {+ _meiliIndex: true,+ _meiliIndexSchemaVersion: { $ne: MEILI_INDEX_SCHEMA_VERSION },+ },+ ],+ },+ ],};try {@@ -548,7 +583,13 @@ const createMeiliMongooseModel = ({// original conversation/message timestamps (fixes sidebar chronological sort).await this.updateMany({ _id: { $in: docsIds } },- { $set: { _meiliIndex: true, _meiliCleanupVersion: meiliCleanupVersion } },+ {+ $set: {+ _meiliIndex: true,+ _meiliCleanupVersion: meiliCleanupVersion,+ _meiliIndexSchemaVersion: MEILI_INDEX_SCHEMA_VERSION,+ },+ },{ timestamps: false },);} catch (error) {@@ -912,6 +953,11 @@ export default function mongoMeili(schema: Schema, options: MongoMeiliOptions):required: false,select: false,},+ _meiliIndexSchemaVersion: {+ type: Number,+ required: false,+ select: false,+ },_meiliCleanupVersion: {type: Number,required: false,