Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🔎 fix: Include Legacy Documents With Undefined `_meiliIndex` in Search Sync (#11745)

Andrei Blizorukov793ddbce9f66
diff --git a/api/db/utils.js b/api/db/utils.js
index 4a311d9832..32051be78d 100644
--- a/api/db/utils.js
+++ b/api/db/utils.js
@@ -26,7 +26,7 @@ async function batchResetMeiliFlags(collection) {
try {
while (hasMore) {
const docs = await collection
- .find({ expiredAt: null, _meiliIndex: true }, { projection: { _id: 1 } })
+ .find({ expiredAt: null, _meiliIndex: { $ne: false } }, { projection: { _id: 1 } })
.limit(BATCH_SIZE)
.toArray();
diff --git a/api/db/utils.spec.js b/api/db/utils.spec.js
index 8b32b4aea8..adf4f6cd86 100644
--- a/api/db/utils.spec.js
+++ b/api/db/utils.spec.js
@@ -265,8 +265,8 @@ describe('batchResetMeiliFlags', () => {
const result = await batchResetMeiliFlags(testCollection);
- // Only one document has _meiliIndex: true
- expect(result).toBe(1);
+ // both documents should be updated
+ expect(result).toBe(2);
});
it('should handle mixed document states correctly', async () => {
@@ -275,16 +275,18 @@ describe('batchResetMeiliFlags', () => {
{ _id: new mongoose.Types.ObjectId(), expiredAt: null, _meiliIndex: false },
{ _id: new mongoose.Types.ObjectId(), expiredAt: new Date(), _meiliIndex: true },
{ _id: new mongoose.Types.ObjectId(), expiredAt: null, _meiliIndex: true },
+ { _id: new mongoose.Types.ObjectId(), expiredAt: null, _meiliIndex: null },
+ { _id: new mongoose.Types.ObjectId(), expiredAt: null },
]);
const result = await batchResetMeiliFlags(testCollection);
- expect(result).toBe(2);
+ expect(result).toBe(4);
const flaggedDocs = await testCollection
.find({ expiredAt: null, _meiliIndex: false })
.toArray();
- expect(flaggedDocs).toHaveLength(3); // 2 were updated, 1 was already false
+ expect(flaggedDocs).toHaveLength(5); // 4 were updated, 1 was already false
});
});
diff --git a/packages/data-schemas/src/models/plugins/mongoMeili.spec.ts b/packages/data-schemas/src/models/plugins/mongoMeili.spec.ts
index a289b88fe0..d988624d13 100644
--- a/packages/data-schemas/src/models/plugins/mongoMeili.spec.ts
+++ b/packages/data-schemas/src/models/plugins/mongoMeili.spec.ts
@@ -1015,4 +1015,239 @@ describe('Meilisearch Mongoose plugin', () => {
]);
});
});
+
+ describe('Missing _meiliIndex property handling in sync process', () => {
+ test('syncWithMeili includes documents with missing _meiliIndex', async () => {
+ const conversationModel = createConversationModel(mongoose) as SchemaWithMeiliMethods;
+ await conversationModel.deleteMany({});
+ mockAddDocumentsInBatches.mockClear();
+
+ // Insert documents with different _meiliIndex states
+ await conversationModel.collection.insertMany([
+ {
+ conversationId: new mongoose.Types.ObjectId(),
+ user: new mongoose.Types.ObjectId(),
+ title: 'Missing _meiliIndex',
+ endpoint: EModelEndpoint.openAI,
+ expiredAt: null,
+ // _meiliIndex is not set (missing/undefined)
+ },
+ {
+ conversationId: new mongoose.Types.ObjectId(),
+ user: new mongoose.Types.ObjectId(),
+ title: 'Explicit false',
+ endpoint: EModelEndpoint.openAI,
+ expiredAt: null,
+ _meiliIndex: false,
+ },
+ {
+ conversationId: new mongoose.Types.ObjectId(),
+ user: new mongoose.Types.ObjectId(),
+ title: 'Already indexed',
+ endpoint: EModelEndpoint.openAI,
+ expiredAt: null,
+ _meiliIndex: true,
+ },
+ ]);
+
+ // Run sync
+ await conversationModel.syncWithMeili();
+
+ // Should have processed 2 documents (missing and false, but not true)
+ expect(mockAddDocumentsInBatches).toHaveBeenCalled();
+
+ // Check that both documents without _meiliIndex=true are now indexed
+ const indexedCount = await conversationModel.countDocuments({
+ expiredAt: null,
+ _meiliIndex: true,
+ });
+ expect(indexedCount).toBe(3); // All 3 should now be indexed
+
+ // Verify documents with missing _meiliIndex were updated
+ const docsWithMissingIndex = await conversationModel.countDocuments({
+ expiredAt: null,
+ title: 'Missing _meiliIndex',
+ _meiliIndex: true,
+ });
+ expect(docsWithMissingIndex).toBe(1);
+ });
+
+ test('getSyncProgress counts documents with missing _meiliIndex as not indexed', async () => {
+ const messageModel = createMessageModel(mongoose) as SchemaWithMeiliMethods;
+ await messageModel.deleteMany({});
+
+ // Insert documents with different _meiliIndex states
+ await messageModel.collection.insertMany([
+ {
+ messageId: new mongoose.Types.ObjectId(),
+ conversationId: new mongoose.Types.ObjectId(),
+ user: new mongoose.Types.ObjectId(),
+ isCreatedByUser: true,
+ expiredAt: null,
+ _meiliIndex: true,
+ },
+ {
+ messageId: new mongoose.Types.ObjectId(),
+ conversationId: new mongoose.Types.ObjectId(),
+ user: new mongoose.Types.ObjectId(),
+ isCreatedByUser: true,
+ expiredAt: null,
+ _meiliIndex: false,
+ },
+ {
+ messageId: new mongoose.Types.ObjectId(),
+ conversationId: new mongoose.Types.ObjectId(),
+ user: new mongoose.Types.ObjectId(),
+ isCreatedByUser: true,
+ expiredAt: null,
+ // _meiliIndex is missing
+ },
+ ]);
+
+ const progress = await messageModel.getSyncProgress();
+
+ // Total should be 3
+ expect(progress.totalDocuments).toBe(3);
+ // Only 1 is indexed (with _meiliIndex: true)
+ expect(progress.totalProcessed).toBe(1);
+ // Not complete since 2 documents are not indexed
+ expect(progress.isComplete).toBe(false);
+ });
+
+ test('query with _meiliIndex: { $ne: true } includes missing values', async () => {
+ const conversationModel = createConversationModel(mongoose) as SchemaWithMeiliMethods;
+ await conversationModel.deleteMany({});
+
+ // Insert documents with different _meiliIndex states
+ await conversationModel.collection.insertMany([
+ {
+ conversationId: new mongoose.Types.ObjectId(),
+ user: new mongoose.Types.ObjectId(),
+ title: 'Missing',
+ endpoint: EModelEndpoint.openAI,
+ expiredAt: null,
+ // _meiliIndex is missing
+ },
+ {
+ conversationId: new mongoose.Types.ObjectId(),
+ user: new mongoose.Types.ObjectId(),
+ title: 'False',
+ endpoint: EModelEndpoint.openAI,
+ expiredAt: null,
+ _meiliIndex: false,
+ },
+ {
+ conversationId: new mongoose.Types.ObjectId(),
+ user: new mongoose.Types.ObjectId(),
+ title: 'True',
+ endpoint: EModelEndpoint.openAI,
+ expiredAt: null,
+ _meiliIndex: true,
+ },
+ ]);
+
+ // Query for documents where _meiliIndex is not true (used in syncWithMeili)
+ const unindexedDocs = await conversationModel.find({
+ expiredAt: null,
+ _meiliIndex: { $ne: true },
+ });
+
+ // Should find 2 documents (missing and false, but not true)
+ expect(unindexedDocs.length).toBe(2);
+ const titles = unindexedDocs.map((doc) => doc.title).sort();
+ expect(titles).toEqual(['False', 'Missing']);
+ });
+
+ test('syncWithMeili processes all documents where _meiliIndex is not true', async () => {
+ const messageModel = createMessageModel(mongoose) as SchemaWithMeiliMethods;
+ await messageModel.deleteMany({});
+ mockAddDocumentsInBatches.mockClear();
+
+ // Create a mix of documents with missing and false _meiliIndex
+ await messageModel.collection.insertMany([
+ {
+ messageId: new mongoose.Types.ObjectId(),
+ conversationId: new mongoose.Types.ObjectId(),
+ user: new mongoose.Types.ObjectId(),
+ isCreatedByUser: true,
+ expiredAt: null,
+ // _meiliIndex missing
+ },
+ {
+ messageId: new mongoose.Types.ObjectId(),
+ conversationId: new mongoose.Types.ObjectId(),
+ user: new mongoose.Types.ObjectId(),
+ isCreatedByUser: true,
+ expiredAt: null,
+ _meiliIndex: false,
+ },
+ {
+ messageId: new mongoose.Types.ObjectId(),
+ conversationId: new mongoose.Types.ObjectId(),
+ user: new mongoose.Types.ObjectId(),
+ isCreatedByUser: true,
+ expiredAt: null,
+ // _meiliIndex missing
+ },
+ ]);
+
+ // Count documents that should be synced (where _meiliIndex: { $ne: true })
+ const toSyncCount = await messageModel.countDocuments({
+ expiredAt: null,
+ _meiliIndex: { $ne: true },
+ });
+ expect(toSyncCount).toBe(3); // All 3 should be synced
+
+ await messageModel.syncWithMeili();
+
+ // All should now be indexed
+ const indexedCount = await messageModel.countDocuments({
+ expiredAt: null,
+ _meiliIndex: true,
+ });
+ expect(indexedCount).toBe(3);
+ });
+
+ test('syncWithMeili treats missing _meiliIndex same as false', async () => {
+ const conversationModel = createConversationModel(mongoose) as SchemaWithMeiliMethods;
+ await conversationModel.deleteMany({});
+ mockAddDocumentsInBatches.mockClear();
+
+ // Insert one document with missing _meiliIndex and one with false
+ await conversationModel.collection.insertMany([
+ {
+ conversationId: new mongoose.Types.ObjectId(),
+ user: new mongoose.Types.ObjectId(),
+ title: 'Missing',
+ endpoint: EModelEndpoint.openAI,
+ expiredAt: null,
+ // _meiliIndex is missing
+ },
+ {
+ conversationId: new mongoose.Types.ObjectId(),
+ user: new mongoose.Types.ObjectId(),
+ title: 'False',
+ endpoint: EModelEndpoint.openAI,
+ expiredAt: null,
+ _meiliIndex: false,
+ },
+ ]);
+
+ // Both should be picked up by the sync query
+ const toSync = await conversationModel.find({
+ expiredAt: null,
+ _meiliIndex: { $ne: true },
+ });
+ expect(toSync.length).toBe(2);
+
+ await conversationModel.syncWithMeili();
+
+ // Both should be indexed after sync
+ const afterSync = await conversationModel.find({
+ expiredAt: null,
+ _meiliIndex: true,
+ });
+ expect(afterSync.length).toBe(2);
+ });
+ });
});
diff --git a/packages/data-schemas/src/models/plugins/mongoMeili.ts b/packages/data-schemas/src/models/plugins/mongoMeili.ts
index 92fc5f328c..1cbe0d8761 100644
--- a/packages/data-schemas/src/models/plugins/mongoMeili.ts
+++ b/packages/data-schemas/src/models/plugins/mongoMeili.ts
@@ -162,8 +162,8 @@ const createMeiliMongooseModel = ({
/**
* Synchronizes data between the MongoDB collection and the MeiliSearch index by
- * incrementally indexing only documents where `expiredAt` is `null` and `_meiliIndex` is `false`
- * (i.e., non-expired documents that have not yet been indexed).
+ * incrementally indexing only documents where `expiredAt` is `null` and `_meiliIndex` is not `true`
+ * (i.e., non-expired documents that have not yet been indexed, including those with missing or null `_meiliIndex`).
* */
static async syncWithMeili(this: SchemaWithMeiliMethods): Promise<void> {
const startTime = Date.now();
@@ -196,7 +196,7 @@ const createMeiliMongooseModel = ({
while (hasMore) {
const query: FilterQuery<unknown> = {
expiredAt: null,
- _meiliIndex: false,
+ _meiliIndex: { $ne: true },
};
try {