Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

📊 fix: MeiliSearch Sync Threshold & Document Count Accuracy (#11406)

Andrei Blizorukov4a1d2b0d941a
diff --git a/api/db/indexSync.js b/api/db/indexSync.js
index b39f018b3a..8e8e999d92 100644
--- a/api/db/indexSync.js
+++ b/api/db/indexSync.js
@@ -13,6 +13,11 @@ const searchEnabled = isEnabled(process.env.SEARCH);
const indexingDisabled = isEnabled(process.env.MEILI_NO_SYNC);
let currentTimeout = null;
+const defaultSyncThreshold = 1000;
+const syncThreshold = process.env.MEILI_SYNC_THRESHOLD
+ ? parseInt(process.env.MEILI_SYNC_THRESHOLD, 10)
+ : defaultSyncThreshold;
+
class MeiliSearchClient {
static instance = null;
@@ -221,25 +226,25 @@ async function performSync(flowManager, flowId, flowType) {
}
// Check if we need to sync messages
+ logger.info('[indexSync] Requesting message sync progress...');
const messageProgress = await Message.getSyncProgress();
if (!messageProgress.isComplete || settingsUpdated) {
logger.info(
`[indexSync] Messages need syncing: ${messageProgress.totalProcessed}/${messageProgress.totalDocuments} indexed`,
);
- // Check if we should do a full sync or incremental
- const messageCount = await Message.countDocuments();
+ const messageCount = messageProgress.totalDocuments;
const messagesIndexed = messageProgress.totalProcessed;
- const syncThreshold = parseInt(process.env.MEILI_SYNC_THRESHOLD || '1000', 10);
+ const unindexedMessages = messageCount - messagesIndexed;
- if (messageCount - messagesIndexed > syncThreshold) {
- logger.info('[indexSync] Starting full message sync due to large difference');
- await Message.syncWithMeili();
- messagesSync = true;
- } else if (messageCount !== messagesIndexed) {
- logger.warn('[indexSync] Messages out of sync, performing incremental sync');
+ if (settingsUpdated || unindexedMessages > syncThreshold) {
+ logger.info(`[indexSync] Starting message sync (${unindexedMessages} unindexed)`);
await Message.syncWithMeili();
messagesSync = true;
+ } else if (unindexedMessages > 0) {
+ logger.info(
+ `[indexSync] ${unindexedMessages} messages unindexed (below threshold: ${syncThreshold}, skipping)`,
+ );
}
} else {
logger.info(
@@ -254,18 +259,18 @@ async function performSync(flowManager, flowId, flowType) {
`[indexSync] Conversations need syncing: ${convoProgress.totalProcessed}/${convoProgress.totalDocuments} indexed`,
);
- const convoCount = await Conversation.countDocuments();
+ const convoCount = convoProgress.totalDocuments;
const convosIndexed = convoProgress.totalProcessed;
- const syncThreshold = parseInt(process.env.MEILI_SYNC_THRESHOLD || '1000', 10);
- if (convoCount - convosIndexed > syncThreshold) {
- logger.info('[indexSync] Starting full conversation sync due to large difference');
- await Conversation.syncWithMeili();
- convosSync = true;
- } else if (convoCount !== convosIndexed) {
- logger.warn('[indexSync] Convos out of sync, performing incremental sync');
+ const unindexedConvos = convoCount - convosIndexed;
+ if (settingsUpdated || unindexedConvos > syncThreshold) {
+ logger.info(`[indexSync] Starting convos sync (${unindexedConvos} unindexed)`);
await Conversation.syncWithMeili();
convosSync = true;
+ } else if (unindexedConvos > 0) {
+ logger.info(
+ `[indexSync] ${unindexedConvos} convos unindexed (below threshold: ${syncThreshold}, skipping)`,
+ );
}
} else {
logger.info(
diff --git a/api/db/indexSync.spec.js b/api/db/indexSync.spec.js
new file mode 100644
index 0000000000..c2e5901d6a
--- /dev/null
+++ b/api/db/indexSync.spec.js
@@ -0,0 +1,465 @@
+/**
+ * Unit tests for performSync() function in indexSync.js
+ *
+ * Tests use real mongoose with mocked model methods, only mocking external calls.
+ */
+
+const mongoose = require('mongoose');
+
+// Mock only external dependencies (not internal classes/models)
+const mockLogger = {
+ info: jest.fn(),
+ warn: jest.fn(),
+ error: jest.fn(),
+ debug: jest.fn(),
+};
+
+const mockMeiliHealth = jest.fn();
+const mockMeiliIndex = jest.fn();
+const mockBatchResetMeiliFlags = jest.fn();
+const mockIsEnabled = jest.fn();
+const mockGetLogStores = jest.fn();
+
+// Create mock models that will be reused
+const createMockModel = (collectionName) => ({
+ collection: { name: collectionName },
+ getSyncProgress: jest.fn(),
+ syncWithMeili: jest.fn(),
+ countDocuments: jest.fn(),
+});
+
+const originalMessageModel = mongoose.models.Message;
+const originalConversationModel = mongoose.models.Conversation;
+
+// Mock external modules
+jest.mock('@librechat/data-schemas', () => ({
+ logger: mockLogger,
+}));
+
+jest.mock('meilisearch', () => ({
+ MeiliSearch: jest.fn(() => ({
+ health: mockMeiliHealth,
+ index: mockMeiliIndex,
+ })),
+}));
+
+jest.mock('./utils', () => ({
+ batchResetMeiliFlags: mockBatchResetMeiliFlags,
+}));
+
+jest.mock('@librechat/api', () => ({
+ isEnabled: mockIsEnabled,
+ FlowStateManager: jest.fn(),
+}));
+
+jest.mock('~/cache', () => ({
+ getLogStores: mockGetLogStores,
+}));
+
+// Set environment before module load
+process.env.MEILI_HOST = 'http://localhost:7700';
+process.env.MEILI_MASTER_KEY = 'test-key';
+process.env.SEARCH = 'true';
+process.env.MEILI_SYNC_THRESHOLD = '1000'; // Set threshold before module loads
+
+describe('performSync() - syncThreshold logic', () => {
+ const ORIGINAL_ENV = process.env;
+ let Message;
+ let Conversation;
+
+ beforeAll(() => {
+ Message = createMockModel('messages');
+ Conversation = createMockModel('conversations');
+
+ mongoose.models.Message = Message;
+ mongoose.models.Conversation = Conversation;
+ });
+
+ beforeEach(() => {
+ // Reset all mocks
+ jest.clearAllMocks();
+ // Reset modules to ensure fresh load of indexSync.js and its top-level consts (like syncThreshold)
+ jest.resetModules();
+
+ // Set up environment
+ process.env = { ...ORIGINAL_ENV };
+ process.env.MEILI_HOST = 'http://localhost:7700';
+ process.env.MEILI_MASTER_KEY = 'test-key';
+ process.env.SEARCH = 'true';
+ delete process.env.MEILI_NO_SYNC;
+
+ // Re-ensure models are available in mongoose after resetModules
+ // We must require mongoose again to get the fresh instance that indexSync will use
+ const mongoose = require('mongoose');
+ mongoose.models.Message = Message;
+ mongoose.models.Conversation = Conversation;
+
+ // Mock isEnabled
+ mockIsEnabled.mockImplementation((val) => val === 'true' || val === true);
+
+ // Mock MeiliSearch client responses
+ mockMeiliHealth.mockResolvedValue({ status: 'available' });
+ mockMeiliIndex.mockReturnValue({
+ getSettings: jest.fn().mockResolvedValue({ filterableAttributes: ['user'] }),
+ updateSettings: jest.fn().mockResolvedValue({}),
+ search: jest.fn().mockResolvedValue({ hits: [] }),
+ });
+
+ mockBatchResetMeiliFlags.mockResolvedValue(undefined);
+ });
+
+ afterEach(() => {
+ process.env = ORIGINAL_ENV;
+ });
+
+ afterAll(() => {
+ mongoose.models.Message = originalMessageModel;
+ mongoose.models.Conversation = originalConversationModel;
+ });
+
+ test('triggers sync when unindexed messages exceed syncThreshold', async () => {
+ // Arrange: Set threshold before module load
+ process.env.MEILI_SYNC_THRESHOLD = '1000';
+
+ // Arrange: 1050 unindexed messages > 1000 threshold
+ Message.getSyncProgress.mockResolvedValue({
+ totalProcessed: 100,
+ totalDocuments: 1150, // 1050 unindexed
+ isComplete: false,
+ });
+
+ Conversation.getSyncProgress.mockResolvedValue({
+ totalProcessed: 50,
+ totalDocuments: 50,
+ isComplete: true,
+ });
+
+ Message.syncWithMeili.mockResolvedValue(undefined);
+
+ // Act
+ const indexSync = require('./indexSync');
+ await indexSync();
+
+ // Assert: No countDocuments calls
+ expect(Message.countDocuments).not.toHaveBeenCalled();
+ expect(Conversation.countDocuments).not.toHaveBeenCalled();
+
+ // Assert: Message sync triggered because 1050 > 1000
+ expect(Message.syncWithMeili).toHaveBeenCalledTimes(1);
+ expect(mockLogger.info).toHaveBeenCalledWith(
+ '[indexSync] Messages need syncing: 100/1150 indexed',
+ );
+ expect(mockLogger.info).toHaveBeenCalledWith(
+ '[indexSync] Starting message sync (1050 unindexed)',
+ );
+
+ // Assert: Conversation sync NOT triggered (already complete)
+ expect(Conversation.syncWithMeili).not.toHaveBeenCalled();
+ });
+
+ test('skips sync when unindexed messages are below syncThreshold', async () => {
+ // Arrange: 50 unindexed messages < 1000 threshold
+ Message.getSyncProgress.mockResolvedValue({
+ totalProcessed: 100,
+ totalDocuments: 150, // 50 unindexed
+ isComplete: false,
+ });
+
+ Conversation.getSyncProgress.mockResolvedValue({
+ totalProcessed: 50,
+ totalDocuments: 50,
+ isComplete: true,
+ });
+
+ process.env.MEILI_SYNC_THRESHOLD = '1000';
+
+ // Act
+ const indexSync = require('./indexSync');
+ await indexSync();
+
+ // Assert: No countDocuments calls
+ expect(Message.countDocuments).not.toHaveBeenCalled();
+ expect(Conversation.countDocuments).not.toHaveBeenCalled();
+
+ // Assert: Message sync NOT triggered because 50 < 1000
+ expect(Message.syncWithMeili).not.toHaveBeenCalled();
+ expect(mockLogger.info).toHaveBeenCalledWith(
+ '[indexSync] Messages need syncing: 100/150 indexed',
+ );
+ expect(mockLogger.info).toHaveBeenCalledWith(
+ '[indexSync] 50 messages unindexed (below threshold: 1000, skipping)',
+ );
+
+ // Assert: Conversation sync NOT triggered (already complete)
+ expect(Conversation.syncWithMeili).not.toHaveBeenCalled();
+ });
+
+ test('respects syncThreshold at boundary (exactly at threshold)', async () => {
+ // Arrange: 1000 unindexed messages = 1000 threshold (NOT greater than)
+ Message.getSyncProgress.mockResolvedValue({
+ totalProcessed: 100,
+ totalDocuments: 1100, // 1000 unindexed
+ isComplete: false,
+ });
+
+ Conversation.getSyncProgress.mockResolvedValue({
+ totalProcessed: 0,
+ totalDocuments: 0,
+ isComplete: true,
+ });
+
+ process.env.MEILI_SYNC_THRESHOLD = '1000';
+
+ // Act
+ const indexSync = require('./indexSync');
+ await indexSync();
+
+ // Assert: No countDocuments calls
+ expect(Message.countDocuments).not.toHaveBeenCalled();
+
+ // Assert: Message sync NOT triggered because 1000 is NOT > 1000
+ expect(Message.syncWithMeili).not.toHaveBeenCalled();
+ expect(mockLogger.info).toHaveBeenCalledWith(
+ '[indexSync] Messages need syncing: 100/1100 indexed',
+ );
+ expect(mockLogger.info).toHaveBeenCalledWith(
+ '[indexSync] 1000 messages unindexed (below threshold: 1000, skipping)',
+ );
+ });
+
+ test('triggers sync when unindexed is threshold + 1', async () => {
+ // Arrange: 1001 unindexed messages > 1000 threshold
+ Message.getSyncProgress.mockResolvedValue({
+ totalProcessed: 100,
+ totalDocuments: 1101, // 1001 unindexed
+ isComplete: false,
+ });
+
+ Conversation.getSyncProgress.mockResolvedValue({
+ totalProcessed: 0,
+ totalDocuments: 0,
+ isComplete: true,
+ });
+
+ Message.syncWithMeili.mockResolvedValue(undefined);
+
+ process.env.MEILI_SYNC_THRESHOLD = '1000';
+
+ // Act
+ const indexSync = require('./indexSync');
+ await indexSync();
+
+ // Assert: No countDocuments calls
+ expect(Message.countDocuments).not.toHaveBeenCalled();
+
+ // Assert: Message sync triggered because 1001 > 1000
+ expect(Message.syncWithMeili).toHaveBeenCalledTimes(1);
+ expect(mockLogger.info).toHaveBeenCalledWith(
+ '[indexSync] Messages need syncing: 100/1101 indexed',
+ );
+ expect(mockLogger.info).toHaveBeenCalledWith(
+ '[indexSync] Starting message sync (1001 unindexed)',
+ );
+ });
+
+ test('uses totalDocuments from convoProgress for conversation sync decisions', async () => {
+ // Arrange: Messages complete, conversations need sync
+ Message.getSyncProgress.mockResolvedValue({
+ totalProcessed: 100,
+ totalDocuments: 100,
+ isComplete: true,
+ });
+
+ Conversation.getSyncProgress.mockResolvedValue({
+ totalProcessed: 50,
+ totalDocuments: 1100, // 1050 unindexed > 1000 threshold
+ isComplete: false,
+ });
+
+ Conversation.syncWithMeili.mockResolvedValue(undefined);
+
+ process.env.MEILI_SYNC_THRESHOLD = '1000';
+
+ // Act
+ const indexSync = require('./indexSync');
+ await indexSync();
+
+ // Assert: No countDocuments calls (the optimization)
+ expect(Message.countDocuments).not.toHaveBeenCalled();
+ expect(Conversation.countDocuments).not.toHaveBeenCalled();
+
+ // Assert: Only conversation sync triggered
+ expect(Message.syncWithMeili).not.toHaveBeenCalled();
+ expect(Conversation.syncWithMeili).toHaveBeenCalledTimes(1);
+ expect(mockLogger.info).toHaveBeenCalledWith(
+ '[indexSync] Conversations need syncing: 50/1100 indexed',
+ );
+ expect(mockLogger.info).toHaveBeenCalledWith(
+ '[indexSync] Starting convos sync (1050 unindexed)',
+ );
+ });
+
+ test('skips sync when collections are fully synced', async () => {
+ // Arrange: Everything already synced
+ Message.getSyncProgress.mockResolvedValue({
+ totalProcessed: 100,
+ totalDocuments: 100,
+ isComplete: true,
+ });
+
+ Conversation.getSyncProgress.mockResolvedValue({
+ totalProcessed: 50,
+ totalDocuments: 50,
+ isComplete: true,
+ });
+
+ // Act
+ const indexSync = require('./indexSync');
+ await indexSync();
+
+ // Assert: No countDocuments calls
+ expect(Message.countDocuments).not.toHaveBeenCalled();
+ expect(Conversation.countDocuments).not.toHaveBeenCalled();
+
+ // Assert: No sync triggered
+ expect(Message.syncWithMeili).not.toHaveBeenCalled();
+ expect(Conversation.syncWithMeili).not.toHaveBeenCalled();
+
+ // Assert: Correct logs
+ expect(mockLogger.info).toHaveBeenCalledWith('[indexSync] Messages are fully synced: 100/100');
+ expect(mockLogger.info).toHaveBeenCalledWith(
+ '[indexSync] Conversations are fully synced: 50/50',
+ );
+ });
+
+ test('triggers message sync when settingsUpdated even if below syncThreshold', async () => {
+ // Arrange: Only 50 unindexed messages (< 1000 threshold), but settings were updated
+ Message.getSyncProgress.mockResolvedValue({
+ totalProcessed: 100,
+ totalDocuments: 150, // 50 unindexed
+ isComplete: false,
+ });
+
+ Conversation.getSyncProgress.mockResolvedValue({
+ totalProcessed: 50,
+ totalDocuments: 50,
+ isComplete: true,
+ });
+
+ Message.syncWithMeili.mockResolvedValue(undefined);
+
+ // Mock settings update scenario
+ mockMeiliIndex.mockReturnValue({
+ getSettings: jest.fn().mockResolvedValue({ filterableAttributes: [] }), // No user field
+ updateSettings: jest.fn().mockResolvedValue({}),
+ search: jest.fn().mockResolvedValue({ hits: [] }),
+ });
+
+ process.env.MEILI_SYNC_THRESHOLD = '1000';
+
+ // Act
+ const indexSync = require('./indexSync');
+ await indexSync();
+
+ // Assert: Flags were reset due to settings update
+ expect(mockBatchResetMeiliFlags).toHaveBeenCalledWith(Message.collection);
+ expect(mockBatchResetMeiliFlags).toHaveBeenCalledWith(Conversation.collection);
+
+ // Assert: Message sync triggered despite being below threshold (50 < 1000)
+ expect(Message.syncWithMeili).toHaveBeenCalledTimes(1);
+ expect(mockLogger.info).toHaveBeenCalledWith(
+ '[indexSync] Settings updated. Forcing full re-sync to reindex with new configuration...',
+ );
+ expect(mockLogger.info).toHaveBeenCalledWith(
+ '[indexSync] Starting message sync (50 unindexed)',
+ );
+ });
+
+ test('triggers conversation sync when settingsUpdated even if below syncThreshold', async () => {
+ // Arrange: Messages complete, conversations have 50 unindexed (< 1000 threshold), but settings were updated
+ Message.getSyncProgress.mockResolvedValue({
+ totalProcessed: 100,
+ totalDocuments: 100,
+ isComplete: true,
+ });
+
+ Conversation.getSyncProgress.mockResolvedValue({
+ totalProcessed: 50,
+ totalDocuments: 100, // 50 unindexed
+ isComplete: false,
+ });
+
+ Conversation.syncWithMeili.mockResolvedValue(undefined);
+
+ // Mock settings update scenario
+ mockMeiliIndex.mockReturnValue({
+ getSettings: jest.fn().mockResolvedValue({ filterableAttributes: [] }), // No user field
+ updateSettings: jest.fn().mockResolvedValue({}),
+ search: jest.fn().mockResolvedValue({ hits: [] }),
+ });
+
+ process.env.MEILI_SYNC_THRESHOLD = '1000';
+
+ // Act
+ const indexSync = require('./indexSync');
+ await indexSync();
+
+ // Assert: Flags were reset due to settings update
+ expect(mockBatchResetMeiliFlags).toHaveBeenCalledWith(Message.collection);
+ expect(mockBatchResetMeiliFlags).toHaveBeenCalledWith(Conversation.collection);
+
+ // Assert: Conversation sync triggered despite being below threshold (50 < 1000)
+ expect(Conversation.syncWithMeili).toHaveBeenCalledTimes(1);
+ expect(mockLogger.info).toHaveBeenCalledWith(
+ '[indexSync] Settings updated. Forcing full re-sync to reindex with new configuration...',
+ );
+ expect(mockLogger.info).toHaveBeenCalledWith('[indexSync] Starting convos sync (50 unindexed)');
+ });
+
+ test('triggers both message and conversation sync when settingsUpdated even if both below syncThreshold', async () => {
+ // Arrange: Set threshold before module load
+ process.env.MEILI_SYNC_THRESHOLD = '1000';
+
+ // Arrange: Both have documents below threshold (50 each), but settings were updated
+ Message.getSyncProgress.mockResolvedValue({
+ totalProcessed: 100,
+ totalDocuments: 150, // 50 unindexed
+ isComplete: false,
+ });
+
+ Conversation.getSyncProgress.mockResolvedValue({
+ totalProcessed: 50,
+ totalDocuments: 100, // 50 unindexed
+ isComplete: false,
+ });
+
+ Message.syncWithMeili.mockResolvedValue(undefined);
+ Conversation.syncWithMeili.mockResolvedValue(undefined);
+
+ // Mock settings update scenario
+ mockMeiliIndex.mockReturnValue({
+ getSettings: jest.fn().mockResolvedValue({ filterableAttributes: [] }), // No user field
+ updateSettings: jest.fn().mockResolvedValue({}),
+ search: jest.fn().mockResolvedValue({ hits: [] }),
+ });
+
+ // Act
+ const indexSync = require('./indexSync');
+ await indexSync();
+
+ // Assert: Flags were reset due to settings update
+ expect(mockBatchResetMeiliFlags).toHaveBeenCalledWith(Message.collection);
+ expect(mockBatchResetMeiliFlags).toHaveBeenCalledWith(Conversation.collection);
+
+ // Assert: Both syncs triggered despite both being below threshold
+ expect(Message.syncWithMeili).toHaveBeenCalledTimes(1);
+ expect(Conversation.syncWithMeili).toHaveBeenCalledTimes(1);
+ expect(mockLogger.info).toHaveBeenCalledWith(
+ '[indexSync] Settings updated. Forcing full re-sync to reindex with new configuration...',
+ );
+ expect(mockLogger.info).toHaveBeenCalledWith(
+ '[indexSync] Starting message sync (50 unindexed)',
+ );
+ expect(mockLogger.info).toHaveBeenCalledWith('[indexSync] Starting convos sync (50 unindexed)');
+ });
+});