Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

📊 refactor: Use Estimated Document Count for Meilisearch Sync (#11329)

Andrei Blizorukov10f591ab1cce
diff --git a/packages/data-schemas/src/models/plugins/mongoMeili.spec.ts b/packages/data-schemas/src/models/plugins/mongoMeili.spec.ts
index 0bf3ab75ef..8f4ee87aaf 100644
--- a/packages/data-schemas/src/models/plugins/mongoMeili.spec.ts
+++ b/packages/data-schemas/src/models/plugins/mongoMeili.spec.ts
@@ -129,4 +129,139 @@ describe('Meilisearch Mongoose plugin', () => {
expect(mockAddDocuments).not.toHaveBeenCalled();
});
+
+ describe('estimatedDocumentCount usage in syncWithMeili', () => {
+ test('syncWithMeili completes successfully with estimatedDocumentCount', async () => {
+ // Clear any previous documents
+ const conversationModel = createConversationModel(mongoose) as SchemaWithMeiliMethods;
+ await conversationModel.deleteMany({});
+
+ // Create test documents
+ await conversationModel.create({
+ conversationId: new mongoose.Types.ObjectId(),
+ user: new mongoose.Types.ObjectId(),
+ title: 'Test Conversation 1',
+ endpoint: EModelEndpoint.openAI,
+ });
+
+ await conversationModel.create({
+ conversationId: new mongoose.Types.ObjectId(),
+ user: new mongoose.Types.ObjectId(),
+ title: 'Test Conversation 2',
+ endpoint: EModelEndpoint.openAI,
+ });
+
+ // Trigger sync - should use estimatedDocumentCount internally
+ await expect(conversationModel.syncWithMeili()).resolves.not.toThrow();
+
+ // Verify documents were processed
+ expect(mockAddDocuments).toHaveBeenCalled();
+ });
+
+ test('syncWithMeili handles empty collection correctly', async () => {
+ const messageModel = createMessageModel(mongoose) as SchemaWithMeiliMethods;
+ await messageModel.deleteMany({});
+
+ // Verify collection is empty
+ const count = await messageModel.estimatedDocumentCount();
+ expect(count).toBe(0);
+
+ // Sync should complete without error even with 0 estimated documents
+ await expect(messageModel.syncWithMeili()).resolves.not.toThrow();
+ });
+
+ test('estimatedDocumentCount returns count for non-empty collection', async () => {
+ const conversationModel = createConversationModel(mongoose) as SchemaWithMeiliMethods;
+ await conversationModel.deleteMany({});
+
+ // Create documents
+ await conversationModel.create({
+ conversationId: new mongoose.Types.ObjectId(),
+ user: new mongoose.Types.ObjectId(),
+ title: 'Test 1',
+ endpoint: EModelEndpoint.openAI,
+ });
+
+ await conversationModel.create({
+ conversationId: new mongoose.Types.ObjectId(),
+ user: new mongoose.Types.ObjectId(),
+ title: 'Test 2',
+ endpoint: EModelEndpoint.openAI,
+ });
+
+ const estimatedCount = await conversationModel.estimatedDocumentCount();
+ expect(estimatedCount).toBeGreaterThanOrEqual(2);
+ });
+
+ test('estimatedDocumentCount is available on model', async () => {
+ const messageModel = createMessageModel(mongoose) as SchemaWithMeiliMethods;
+
+ // Verify the method exists and is callable
+ expect(typeof messageModel.estimatedDocumentCount).toBe('function');
+
+ // Should be able to call it
+ const result = await messageModel.estimatedDocumentCount();
+ expect(typeof result).toBe('number');
+ expect(result).toBeGreaterThanOrEqual(0);
+ });
+
+ test('syncWithMeili handles mix of syncable and TTL documents correctly', async () => {
+ const messageModel = createMessageModel(mongoose) as SchemaWithMeiliMethods;
+ await messageModel.deleteMany({});
+ mockAddDocuments.mockClear();
+
+ // Create syncable documents (expiredAt: null)
+ await messageModel.create({
+ messageId: new mongoose.Types.ObjectId(),
+ conversationId: new mongoose.Types.ObjectId(),
+ user: new mongoose.Types.ObjectId(),
+ isCreatedByUser: true,
+ expiredAt: null,
+ });
+
+ await messageModel.create({
+ messageId: new mongoose.Types.ObjectId(),
+ conversationId: new mongoose.Types.ObjectId(),
+ user: new mongoose.Types.ObjectId(),
+ isCreatedByUser: false,
+ expiredAt: null,
+ });
+
+ // Create TTL documents (expiredAt set to a date)
+ await messageModel.create({
+ messageId: new mongoose.Types.ObjectId(),
+ conversationId: new mongoose.Types.ObjectId(),
+ user: new mongoose.Types.ObjectId(),
+ isCreatedByUser: true,
+ expiredAt: new Date(),
+ });
+
+ await messageModel.create({
+ messageId: new mongoose.Types.ObjectId(),
+ conversationId: new mongoose.Types.ObjectId(),
+ user: new mongoose.Types.ObjectId(),
+ isCreatedByUser: false,
+ expiredAt: new Date(),
+ });
+
+ // estimatedDocumentCount should count all documents (both syncable and TTL)
+ const estimatedCount = await messageModel.estimatedDocumentCount();
+ expect(estimatedCount).toBe(4);
+
+ // Actual syncable documents (expiredAt: null)
+ const syncableCount = await messageModel.countDocuments({ expiredAt: null });
+ expect(syncableCount).toBe(2);
+
+ // Sync should complete successfully even though estimated count is higher than processed count
+ await expect(messageModel.syncWithMeili()).resolves.not.toThrow();
+
+ // Only syncable documents should be indexed (2 documents, not 4)
+ // The mock should be called once per batch, and we have 2 documents
+ expect(mockAddDocuments).toHaveBeenCalled();
+
+ // Verify that only 2 documents were indexed (the syncable ones)
+ const indexedCount = await messageModel.countDocuments({ _meiliIndex: true });
+ expect(indexedCount).toBe(2);
+ });
+ });
});
diff --git a/packages/data-schemas/src/models/plugins/mongoMeili.ts b/packages/data-schemas/src/models/plugins/mongoMeili.ts
index 2d56303395..548a7d2f1a 100644
--- a/packages/data-schemas/src/models/plugins/mongoMeili.ts
+++ b/packages/data-schemas/src/models/plugins/mongoMeili.ts
@@ -189,8 +189,10 @@ const createMeiliMongooseModel = ({
query._id = { $gt: options.resumeFromId };
}
- // Get total count for progress tracking
- const totalCount = await this.countDocuments(query);
+ // Get approximate total count for progress tracking
+ const approxTotalCount = await this.estimatedDocumentCount();
+ logger.info(`[syncWithMeili] Approximate total number of documents to sync: ${approxTotalCount}`);
+
let processedCount = 0;
// First, handle documents that need to be removed from Meili
@@ -239,8 +241,11 @@ const createMeiliMongooseModel = ({
updateOps = [];
// Log progress
- const progress = Math.round((processedCount / totalCount) * 100);
- logger.info(`[syncWithMeili] Progress: ${progress}% (${processedCount}/${totalCount})`);
+ // Calculate percentage based on approximate total count sometimes might lead to more than 100%
+ // the difference is very small and acceptable for progress tracking
+ const percent = Math.round((processedCount / approxTotalCount) * 100);
+ const progress = Math.min(percent, 100);
+ logger.info(`[syncWithMeili] Progress: ${progress}% (count: ${processedCount})`);
// Add delay to prevent overwhelming resources
if (delayMs > 0) {