Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🕵️ refactor: Optimize Message Search Performance (#9818)
Danny Avila57f8b333bc2f
diff --git a/api/db/indexSync.js b/api/db/indexSync.jsindex 945346a906..9a9ed9507a 100644--- a/api/db/indexSync.js+++ b/api/db/indexSync.js@@ -1,10 +1,8 @@const mongoose = require('mongoose');const { MeiliSearch } = require('meilisearch');const { logger } = require('@librechat/data-schemas');-const { FlowStateManager } = require('@librechat/api');const { CacheKeys } = require('librechat-data-provider');--const { isEnabled } = require('~/server/utils');+const { isEnabled, FlowStateManager } = require('@librechat/api');const { getLogStores } = require('~/cache');const Conversation = mongoose.models.Conversation;@@ -31,6 +29,81 @@ class MeiliSearchClient {}}+/**+ * Ensures indexes have proper filterable attributes configured and checks if documents have user field+ * @param {MeiliSearch} client - MeiliSearch client instance+ * @returns {Promise<boolean>} - true if configuration was updated or re-sync is needed+ */+async function ensureFilterableAttributes(client) {+ try {+ // Check and update messages index+ try {+ const messagesIndex = client.index('messages');+ const settings = await messagesIndex.getSettings();++ if (!settings.filterableAttributes || !settings.filterableAttributes.includes('user')) {+ logger.info('[indexSync] Configuring messages index to filter by user...');+ await messagesIndex.updateSettings({+ filterableAttributes: ['user'],+ });+ logger.info('[indexSync] Messages index configured for user filtering');+ logger.info('[indexSync] Index configuration updated. Full re-sync will be triggered.');+ return true;+ }++ // Check if existing documents have user field indexed+ try {+ const searchResult = await messagesIndex.search('', { limit: 1 });+ if (searchResult.hits.length > 0 && !searchResult.hits[0].user) {+ logger.info('[indexSync] Existing messages missing user field, re-sync needed');+ return true;+ }+ } catch (searchError) {+ logger.debug('[indexSync] Could not check message documents:', searchError.message);+ }+ } catch (error) {+ if (error.code !== 'index_not_found') {+ logger.warn('[indexSync] Could not check/update messages index settings:', error.message);+ }+ }++ // Check and update conversations index+ try {+ const convosIndex = client.index('convos');+ const settings = await convosIndex.getSettings();++ if (!settings.filterableAttributes || !settings.filterableAttributes.includes('user')) {+ logger.info('[indexSync] Configuring convos index to filter by user...');+ await convosIndex.updateSettings({+ filterableAttributes: ['user'],+ });+ logger.info('[indexSync] Convos index configured for user filtering');+ logger.info('[indexSync] Index configuration updated. Full re-sync will be triggered.');+ return true;+ }++ // Check if existing documents have user field indexed+ try {+ const searchResult = await convosIndex.search('', { limit: 1 });+ if (searchResult.hits.length > 0 && !searchResult.hits[0].user) {+ logger.info('[indexSync] Existing conversations missing user field, re-sync needed');+ return true;+ }+ } catch (searchError) {+ logger.debug('[indexSync] Could not check conversation documents:', searchError.message);+ }+ } catch (error) {+ if (error.code !== 'index_not_found') {+ logger.warn('[indexSync] Could not check/update convos index settings:', error.message);+ }+ }+ } catch (error) {+ logger.error('[indexSync] Error ensuring filterable attributes:', error);+ }++ return false;+}+/*** Performs the actual sync operations for messages and conversations*/@@ -47,12 +120,27 @@ async function performSync() {return { messagesSync: false, convosSync: false };}+ /** Ensures indexes have proper filterable attributes configured */+ const configUpdated = await ensureFilterableAttributes(client);+let messagesSync = false;let convosSync = false;+ // If configuration was just updated or documents are missing user field, force a full re-sync+ if (configUpdated) {+ logger.info('[indexSync] Forcing full re-sync to ensure user field is properly indexed...');++ // Reset sync flags to force full re-sync+ await Message.collection.updateMany({ _meiliIndex: true }, { $set: { _meiliIndex: false } });+ await Conversation.collection.updateMany(+ { _meiliIndex: true },+ { $set: { _meiliIndex: false } },+ );+ }+// Check if we need to sync messagesconst messageProgress = await Message.getSyncProgress();- if (!messageProgress.isComplete) {+ if (!messageProgress.isComplete || configUpdated) {logger.info(`[indexSync] Messages need syncing: ${messageProgress.totalProcessed}/${messageProgress.totalDocuments} indexed`,);@@ -79,7 +167,7 @@ async function performSync() {// Check if we need to sync conversationsconst convoProgress = await Conversation.getSyncProgress();- if (!convoProgress.isComplete) {+ if (!convoProgress.isComplete || configUpdated) {logger.info(`[indexSync] Conversations need syncing: ${convoProgress.totalProcessed}/${convoProgress.totalDocuments} indexed`,);diff --git a/api/models/Conversation.js b/api/models/Conversation.jsindex 9f7aa90014..13c329aa4a 100644--- a/api/models/Conversation.js+++ b/api/models/Conversation.js@@ -174,7 +174,7 @@ module.exports = {if (search) {try {- const meiliResults = await Conversation.meiliSearch(search);+ const meiliResults = await Conversation.meiliSearch(search, { filter: `user = "${user}"` });const matchingIds = Array.isArray(meiliResults.hits)? meiliResults.hits.map((result) => result.conversationId): [];diff --git a/api/server/routes/messages.js b/api/server/routes/messages.jsindex 0a277a1bd6..f1ac903ece 100644--- a/api/server/routes/messages.js+++ b/api/server/routes/messages.js@@ -3,8 +3,8 @@ const { logger } = require('@librechat/data-schemas');const { ContentTypes } = require('librechat-data-provider');const {saveConvo,- saveMessage,getMessage,+ saveMessage,getMessages,updateMessage,deleteMessages,@@ -58,34 +58,51 @@ router.get('/', async (req, res) => {const nextCursor = messages.length > pageSize ? messages.pop()[sortField] : null;response = { messages, nextCursor };} else if (search) {- const searchResults = await Message.meiliSearch(search, undefined, true);+ const searchResults = await Message.meiliSearch(search, { filter: `user = "${user}"` }, true);const messages = searchResults.hits || [];const result = await getConvosQueried(req.user.id, messages, cursor);- const activeMessages = [];+ const messageIds = [];+ const cleanedMessages = [];for (let i = 0; i < messages.length; i++) {let message = messages[i];if (message.conversationId.includes('--')) {message.conversationId = cleanUpPrimaryKeyValue(message.conversationId);}if (result.convoMap[message.conversationId]) {- const convo = result.convoMap[message.conversationId];-- const dbMessage = await getMessage({ user, messageId: message.messageId });- activeMessages.push({- ...message,- title: convo.title,- conversationId: message.conversationId,- model: convo.model,- isCreatedByUser: dbMessage?.isCreatedByUser,- endpoint: dbMessage?.endpoint,- iconURL: dbMessage?.iconURL,- });+ messageIds.push(message.messageId);+ cleanedMessages.push(message);}}+ const dbMessages = await getMessages({+ user,+ messageId: { $in: messageIds },+ });++ const dbMessageMap = {};+ for (const dbMessage of dbMessages) {+ dbMessageMap[dbMessage.messageId] = dbMessage;+ }++ const activeMessages = [];+ for (const message of cleanedMessages) {+ const convo = result.convoMap[message.conversationId];+ const dbMessage = dbMessageMap[message.messageId];++ activeMessages.push({+ ...message,+ title: convo.title,+ conversationId: message.conversationId,+ model: convo.model,+ isCreatedByUser: dbMessage?.isCreatedByUser,+ endpoint: dbMessage?.endpoint,+ iconURL: dbMessage?.iconURL,+ });+ }+response = { messages: activeMessages, nextCursor: null };} else {response = { messages: [], nextCursor: null };diff --git a/client/src/routes/Search.tsx b/client/src/routes/Search.tsxindex 4e6a180a17..a9f210832c 100644--- a/client/src/routes/Search.tsx+++ b/client/src/routes/Search.tsx@@ -1,6 +1,5 @@import { useEffect, useMemo } from 'react';import { useRecoilValue } from 'recoil';-import { buildTree } from 'librechat-data-provider';import { Spinner, useToastContext } from '@librechat/client';import MinimalMessagesWrapper from '~/components/Chat/Messages/MinimalMessages';import { useNavScrolling, useLocalize, useAuthContext } from '~/hooks';@@ -43,9 +42,20 @@ export default function Search() {});const messages = useMemo(() => {- const msgs = searchMessages?.pages.flatMap((page) => page.messages) || [];- const dataTree = buildTree({ messages: msgs, fileMap });- return dataTree?.length === 0 ? null : (dataTree ?? null);+ const msgs =+ searchMessages?.pages.flatMap((page) =>+ page.messages.map((message) => {+ if (!message.files || !fileMap) {+ return message;+ }+ return {+ ...message,+ files: message.files.map((file) => fileMap[file.file_id ?? ''] ?? file),+ };+ }),+ ) || [];++ return msgs.length === 0 ? null : msgs;}, [fileMap, searchMessages?.pages]);useEffect(() => {diff --git a/packages/data-schemas/src/methods/share.test.ts b/packages/data-schemas/src/methods/share.test.tsindex 45b7faeb1a..1deec967e8 100644--- a/packages/data-schemas/src/methods/share.test.ts+++ b/packages/data-schemas/src/methods/share.test.ts@@ -427,13 +427,14 @@ describe('Share Methods', () => {expect(privateResults.links[0].title).toBe('Private Share');});- test('should handle search with mocked meiliSearch', async () => {+ test('should handle search with mocked meiliSearch and user filter', async () => {const userId = new mongoose.Types.ObjectId().toString();// Mock meiliSearch method- Conversation.meiliSearch = jest.fn().mockResolvedValue({+ const meiliSearchMock = jest.fn().mockResolvedValue({hits: [{ conversationId: 'conv1' }],});+ Conversation.meiliSearch = meiliSearchMock;await SharedLink.create([{@@ -464,6 +465,9 @@ describe('Share Methods', () => {expect(result.links).toHaveLength(1);expect(result.links[0].title).toBe('Matching Share');++ // Verify that meiliSearch was called with the correct user filter+ expect(meiliSearchMock).toHaveBeenCalledWith('search term', { filter: `user = "${userId}"` });});test('should handle empty results', async () => {@@ -475,6 +479,98 @@ describe('Share Methods', () => {expect(result.nextCursor).toBeUndefined();});+ test('should only return shares from search results for the current user', async () => {+ const userId1 = new mongoose.Types.ObjectId().toString();+ const userId2 = new mongoose.Types.ObjectId().toString();++ // Mock meiliSearch to simulate finding conversations from both users+ const meiliSearchMock = jest.fn().mockImplementation((searchTerm, params) => {+ // Simulate MeiliSearch filtering by user+ const filter = params?.filter;+ if (filter && filter.includes(userId1)) {+ return Promise.resolve({+ hits: [{ conversationId: 'conv1' }, { conversationId: 'conv3' }],+ });+ } else if (filter && filter.includes(userId2)) {+ return Promise.resolve({ hits: [{ conversationId: 'conv2' }] });+ }+ // Without filter, would return all conversations (security issue)+ return Promise.resolve({+ hits: [+ { conversationId: 'conv1' },+ { conversationId: 'conv2' },+ { conversationId: 'conv3' },+ ],+ });+ });+ Conversation.meiliSearch = meiliSearchMock;++ // Create shares for different users+ await SharedLink.create([+ {+ shareId: 'share1',+ conversationId: 'conv1',+ user: userId1,+ title: 'User 1 Share',+ isPublic: true,+ },+ {+ shareId: 'share2',+ conversationId: 'conv2',+ user: userId2,+ title: 'User 2 Share',+ isPublic: true,+ },+ {+ shareId: 'share3',+ conversationId: 'conv3',+ user: userId1,+ title: 'Another User 1 Share',+ isPublic: true,+ },+ ]);++ // Search as userId1+ const result1 = await shareMethods.getSharedLinks(+ userId1,+ undefined,+ 10,+ true,+ 'createdAt',+ 'desc',+ 'search term',+ );++ // Should only get shares from conversations belonging to userId1+ expect(result1.links).toHaveLength(2);+ expect(result1.links.every((link) => link.title.includes('User 1'))).toBe(true);++ // Verify correct filter was used+ expect(meiliSearchMock).toHaveBeenCalledWith('search term', {+ filter: `user = "${userId1}"`,+ });++ // Search as userId2+ const result2 = await shareMethods.getSharedLinks(+ userId2,+ undefined,+ 10,+ true,+ 'createdAt',+ 'desc',+ 'search term',+ );++ // Should only get shares from conversations belonging to userId2+ expect(result2.links).toHaveLength(1);+ expect(result2.links[0].title).toBe('User 2 Share');++ // Verify correct filter was used for second user+ expect(meiliSearchMock).toHaveBeenCalledWith('search term', {+ filter: `user = "${userId2}"`,+ });+ });+test('should only return shares for the specified user', async () => {const userId1 = new mongoose.Types.ObjectId().toString();const userId2 = new mongoose.Types.ObjectId().toString();diff --git a/packages/data-schemas/src/methods/share.ts b/packages/data-schemas/src/methods/share.tsindex 7c16ead209..8ff71fd718 100644--- a/packages/data-schemas/src/methods/share.ts+++ b/packages/data-schemas/src/methods/share.ts@@ -150,7 +150,9 @@ export function createShareMethods(mongoose: typeof import('mongoose')) {if (search && search.trim()) {try {- const searchResults = await Conversation.meiliSearch(search);+ const searchResults = await Conversation.meiliSearch(search, {+ filter: `user = "${user}"`,+ });if (!searchResults?.hits?.length) {return {diff --git a/packages/data-schemas/src/models/plugins/mongoMeili.ts b/packages/data-schemas/src/models/plugins/mongoMeili.tsindex c908135433..eacb3f2b60 100644--- a/packages/data-schemas/src/models/plugins/mongoMeili.ts+++ b/packages/data-schemas/src/models/plugins/mongoMeili.ts@@ -1,6 +1,6 @@import _ from 'lodash';import { MeiliSearch } from 'meilisearch';-import type { SearchResponse, Index } from 'meilisearch';+import type { SearchResponse, SearchParams, Index } from 'meilisearch';import type {CallbackWithoutResultAndOptionalError,FilterQuery,@@ -75,7 +75,7 @@ export interface SchemaWithMeiliMethods extends Model<DocumentWithMeiliIndex> {setMeiliIndexSettings(settings: Record<string, unknown>): Promise<unknown>;meiliSearch(q: string,- params?: Record<string, unknown>,+ params?: SearchParams,populate?: boolean,): Promise<SearchResponse<MeiliIndexable, Record<string, unknown>>>;}@@ -386,7 +386,7 @@ const createMeiliMongooseModel = ({static async meiliSearch(this: SchemaWithMeiliMethods,q: string,- params: Record<string, unknown>,+ params: SearchParams,populate: boolean,): Promise<SearchResponse<MeiliIndexable, Record<string, unknown>>> {const data = await index.search(q, params);@@ -644,6 +644,16 @@ export default function mongoMeili(schema: Schema, options: MongoMeiliOptions):logger.error(`[mongoMeili] Error checking index ${indexName}:`, error);}}++ // Configure index settings to make 'user' field filterable+ try {+ await index.updateSettings({+ filterableAttributes: ['user'],+ });+ logger.debug(`[mongoMeili] Updated index ${indexName} settings to make 'user' filterable`);+ } catch (settingsError) {+ logger.error(`[mongoMeili] Error updating index settings for ${indexName}:`, settingsError);+ }})();// Collect attributes from the schema that should be indexed@@ -654,6 +664,13 @@ export default function mongoMeili(schema: Schema, options: MongoMeiliOptions):}, []),];+ // CRITICAL: Always include 'user' field for proper filtering+ // This ensures existing deployments can filter by user after migration+ if (schema.obj.user && !attributesToIndex.includes('user')) {+ attributesToIndex.push('user');+ logger.debug(`[mongoMeili] Added 'user' field to ${indexName} index attributes`);+ }+schema.loadClass(createMeiliMongooseModel({ index, attributesToIndex, syncOptions }));// Register Mongoose hooksdiff --git a/packages/data-schemas/src/schema/convo.ts b/packages/data-schemas/src/schema/convo.tsindex 9680a49d9e..4c3f09373f 100644--- a/packages/data-schemas/src/schema/convo.ts+++ b/packages/data-schemas/src/schema/convo.ts@@ -19,6 +19,7 @@ const convoSchema: Schema<IConversation> = new Schema(user: {type: String,index: true,+ meiliIndex: true,},messages: [{ type: Schema.Types.ObjectId, ref: 'Message' }],agentOptions: {diff --git a/packages/data-schemas/src/schema/message.ts b/packages/data-schemas/src/schema/message.tsindex 15a80ae80e..c11252cb87 100644--- a/packages/data-schemas/src/schema/message.ts+++ b/packages/data-schemas/src/schema/message.ts@@ -21,6 +21,7 @@ const messageSchema: Schema<IMessage> = new Schema(index: true,required: true,default: null,+ meiliIndex: true,},model: {type: String,