Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
📊 refactor: Use Estimated Document Count for Meilisearch Sync (#11329)
Andrei Blizorukov10f591ab1cce
diff --git a/packages/data-schemas/src/models/plugins/mongoMeili.spec.ts b/packages/data-schemas/src/models/plugins/mongoMeili.spec.tsindex 0bf3ab75ef..8f4ee87aaf 100644--- a/packages/data-schemas/src/models/plugins/mongoMeili.spec.ts+++ b/packages/data-schemas/src/models/plugins/mongoMeili.spec.ts@@ -129,4 +129,139 @@ describe('Meilisearch Mongoose plugin', () => {expect(mockAddDocuments).not.toHaveBeenCalled();});++ describe('estimatedDocumentCount usage in syncWithMeili', () => {+ test('syncWithMeili completes successfully with estimatedDocumentCount', async () => {+ // Clear any previous documents+ const conversationModel = createConversationModel(mongoose) as SchemaWithMeiliMethods;+ await conversationModel.deleteMany({});++ // Create test documents+ await conversationModel.create({+ conversationId: new mongoose.Types.ObjectId(),+ user: new mongoose.Types.ObjectId(),+ title: 'Test Conversation 1',+ endpoint: EModelEndpoint.openAI,+ });++ await conversationModel.create({+ conversationId: new mongoose.Types.ObjectId(),+ user: new mongoose.Types.ObjectId(),+ title: 'Test Conversation 2',+ endpoint: EModelEndpoint.openAI,+ });++ // Trigger sync - should use estimatedDocumentCount internally+ await expect(conversationModel.syncWithMeili()).resolves.not.toThrow();++ // Verify documents were processed+ expect(mockAddDocuments).toHaveBeenCalled();+ });++ test('syncWithMeili handles empty collection correctly', async () => {+ const messageModel = createMessageModel(mongoose) as SchemaWithMeiliMethods;+ await messageModel.deleteMany({});++ // Verify collection is empty+ const count = await messageModel.estimatedDocumentCount();+ expect(count).toBe(0);++ // Sync should complete without error even with 0 estimated documents+ await expect(messageModel.syncWithMeili()).resolves.not.toThrow();+ });++ test('estimatedDocumentCount returns count for non-empty collection', async () => {+ const conversationModel = createConversationModel(mongoose) as SchemaWithMeiliMethods;+ await conversationModel.deleteMany({});++ // Create documents+ await conversationModel.create({+ conversationId: new mongoose.Types.ObjectId(),+ user: new mongoose.Types.ObjectId(),+ title: 'Test 1',+ endpoint: EModelEndpoint.openAI,+ });++ await conversationModel.create({+ conversationId: new mongoose.Types.ObjectId(),+ user: new mongoose.Types.ObjectId(),+ title: 'Test 2',+ endpoint: EModelEndpoint.openAI,+ });++ const estimatedCount = await conversationModel.estimatedDocumentCount();+ expect(estimatedCount).toBeGreaterThanOrEqual(2);+ });++ test('estimatedDocumentCount is available on model', async () => {+ const messageModel = createMessageModel(mongoose) as SchemaWithMeiliMethods;++ // Verify the method exists and is callable+ expect(typeof messageModel.estimatedDocumentCount).toBe('function');++ // Should be able to call it+ const result = await messageModel.estimatedDocumentCount();+ expect(typeof result).toBe('number');+ expect(result).toBeGreaterThanOrEqual(0);+ });++ test('syncWithMeili handles mix of syncable and TTL documents correctly', async () => {+ const messageModel = createMessageModel(mongoose) as SchemaWithMeiliMethods;+ await messageModel.deleteMany({});+ mockAddDocuments.mockClear();++ // Create syncable documents (expiredAt: null)+ await messageModel.create({+ messageId: new mongoose.Types.ObjectId(),+ conversationId: new mongoose.Types.ObjectId(),+ user: new mongoose.Types.ObjectId(),+ isCreatedByUser: true,+ expiredAt: null,+ });++ await messageModel.create({+ messageId: new mongoose.Types.ObjectId(),+ conversationId: new mongoose.Types.ObjectId(),+ user: new mongoose.Types.ObjectId(),+ isCreatedByUser: false,+ expiredAt: null,+ });++ // Create TTL documents (expiredAt set to a date)+ await messageModel.create({+ messageId: new mongoose.Types.ObjectId(),+ conversationId: new mongoose.Types.ObjectId(),+ user: new mongoose.Types.ObjectId(),+ isCreatedByUser: true,+ expiredAt: new Date(),+ });++ await messageModel.create({+ messageId: new mongoose.Types.ObjectId(),+ conversationId: new mongoose.Types.ObjectId(),+ user: new mongoose.Types.ObjectId(),+ isCreatedByUser: false,+ expiredAt: new Date(),+ });++ // estimatedDocumentCount should count all documents (both syncable and TTL)+ const estimatedCount = await messageModel.estimatedDocumentCount();+ expect(estimatedCount).toBe(4);++ // Actual syncable documents (expiredAt: null)+ const syncableCount = await messageModel.countDocuments({ expiredAt: null });+ expect(syncableCount).toBe(2);++ // Sync should complete successfully even though estimated count is higher than processed count+ await expect(messageModel.syncWithMeili()).resolves.not.toThrow();++ // Only syncable documents should be indexed (2 documents, not 4)+ // The mock should be called once per batch, and we have 2 documents+ expect(mockAddDocuments).toHaveBeenCalled();++ // Verify that only 2 documents were indexed (the syncable ones)+ const indexedCount = await messageModel.countDocuments({ _meiliIndex: true });+ expect(indexedCount).toBe(2);+ });+ });});diff --git a/packages/data-schemas/src/models/plugins/mongoMeili.ts b/packages/data-schemas/src/models/plugins/mongoMeili.tsindex 2d56303395..548a7d2f1a 100644--- a/packages/data-schemas/src/models/plugins/mongoMeili.ts+++ b/packages/data-schemas/src/models/plugins/mongoMeili.ts@@ -189,8 +189,10 @@ const createMeiliMongooseModel = ({query._id = { $gt: options.resumeFromId };}- // Get total count for progress tracking- const totalCount = await this.countDocuments(query);+ // Get approximate total count for progress tracking+ const approxTotalCount = await this.estimatedDocumentCount();+ logger.info(`[syncWithMeili] Approximate total number of documents to sync: ${approxTotalCount}`);+let processedCount = 0;// First, handle documents that need to be removed from Meili@@ -239,8 +241,11 @@ const createMeiliMongooseModel = ({updateOps = [];// Log progress- const progress = Math.round((processedCount / totalCount) * 100);- logger.info(`[syncWithMeili] Progress: ${progress}% (${processedCount}/${totalCount})`);+ // Calculate percentage based on approximate total count sometimes might lead to more than 100%+ // the difference is very small and acceptable for progress tracking+ const percent = Math.round((processedCount / approxTotalCount) * 100);+ const progress = Math.min(percent, 100);+ logger.info(`[syncWithMeili] Progress: ${progress}% (count: ${processedCount})`);// Add delay to prevent overwhelming resourcesif (delayMs > 0) {