Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🕵️ refactor: Optimize Message Search Performance (#9818)

Danny Avila57f8b333bc2f
diff --git a/api/db/indexSync.js b/api/db/indexSync.js
index 945346a906..9a9ed9507a 100644
--- a/api/db/indexSync.js
+++ b/api/db/indexSync.js
@@ -1,10 +1,8 @@
const mongoose = require('mongoose');
const { MeiliSearch } = require('meilisearch');
const { logger } = require('@librechat/data-schemas');
-const { FlowStateManager } = require('@librechat/api');
const { CacheKeys } = require('librechat-data-provider');
-
-const { isEnabled } = require('~/server/utils');
+const { isEnabled, FlowStateManager } = require('@librechat/api');
const { getLogStores } = require('~/cache');
const Conversation = mongoose.models.Conversation;
@@ -31,6 +29,81 @@ class MeiliSearchClient {
}
}
+/**
+ * Ensures indexes have proper filterable attributes configured and checks if documents have user field
+ * @param {MeiliSearch} client - MeiliSearch client instance
+ * @returns {Promise<boolean>} - true if configuration was updated or re-sync is needed
+ */
+async function ensureFilterableAttributes(client) {
+ try {
+ // Check and update messages index
+ try {
+ const messagesIndex = client.index('messages');
+ const settings = await messagesIndex.getSettings();
+
+ if (!settings.filterableAttributes || !settings.filterableAttributes.includes('user')) {
+ logger.info('[indexSync] Configuring messages index to filter by user...');
+ await messagesIndex.updateSettings({
+ filterableAttributes: ['user'],
+ });
+ logger.info('[indexSync] Messages index configured for user filtering');
+ logger.info('[indexSync] Index configuration updated. Full re-sync will be triggered.');
+ return true;
+ }
+
+ // Check if existing documents have user field indexed
+ try {
+ const searchResult = await messagesIndex.search('', { limit: 1 });
+ if (searchResult.hits.length > 0 && !searchResult.hits[0].user) {
+ logger.info('[indexSync] Existing messages missing user field, re-sync needed');
+ return true;
+ }
+ } catch (searchError) {
+ logger.debug('[indexSync] Could not check message documents:', searchError.message);
+ }
+ } catch (error) {
+ if (error.code !== 'index_not_found') {
+ logger.warn('[indexSync] Could not check/update messages index settings:', error.message);
+ }
+ }
+
+ // Check and update conversations index
+ try {
+ const convosIndex = client.index('convos');
+ const settings = await convosIndex.getSettings();
+
+ if (!settings.filterableAttributes || !settings.filterableAttributes.includes('user')) {
+ logger.info('[indexSync] Configuring convos index to filter by user...');
+ await convosIndex.updateSettings({
+ filterableAttributes: ['user'],
+ });
+ logger.info('[indexSync] Convos index configured for user filtering');
+ logger.info('[indexSync] Index configuration updated. Full re-sync will be triggered.');
+ return true;
+ }
+
+ // Check if existing documents have user field indexed
+ try {
+ const searchResult = await convosIndex.search('', { limit: 1 });
+ if (searchResult.hits.length > 0 && !searchResult.hits[0].user) {
+ logger.info('[indexSync] Existing conversations missing user field, re-sync needed');
+ return true;
+ }
+ } catch (searchError) {
+ logger.debug('[indexSync] Could not check conversation documents:', searchError.message);
+ }
+ } catch (error) {
+ if (error.code !== 'index_not_found') {
+ logger.warn('[indexSync] Could not check/update convos index settings:', error.message);
+ }
+ }
+ } catch (error) {
+ logger.error('[indexSync] Error ensuring filterable attributes:', error);
+ }
+
+ return false;
+}
+
/**
* Performs the actual sync operations for messages and conversations
*/
@@ -47,12 +120,27 @@ async function performSync() {
return { messagesSync: false, convosSync: false };
}
+ /** Ensures indexes have proper filterable attributes configured */
+ const configUpdated = await ensureFilterableAttributes(client);
+
let messagesSync = false;
let convosSync = false;
+ // If configuration was just updated or documents are missing user field, force a full re-sync
+ if (configUpdated) {
+ logger.info('[indexSync] Forcing full re-sync to ensure user field is properly indexed...');
+
+ // Reset sync flags to force full re-sync
+ await Message.collection.updateMany({ _meiliIndex: true }, { $set: { _meiliIndex: false } });
+ await Conversation.collection.updateMany(
+ { _meiliIndex: true },
+ { $set: { _meiliIndex: false } },
+ );
+ }
+
// Check if we need to sync messages
const messageProgress = await Message.getSyncProgress();
- if (!messageProgress.isComplete) {
+ if (!messageProgress.isComplete || configUpdated) {
logger.info(
`[indexSync] Messages need syncing: ${messageProgress.totalProcessed}/${messageProgress.totalDocuments} indexed`,
);
@@ -79,7 +167,7 @@ async function performSync() {
// Check if we need to sync conversations
const convoProgress = await Conversation.getSyncProgress();
- if (!convoProgress.isComplete) {
+ if (!convoProgress.isComplete || configUpdated) {
logger.info(
`[indexSync] Conversations need syncing: ${convoProgress.totalProcessed}/${convoProgress.totalDocuments} indexed`,
);
diff --git a/api/models/Conversation.js b/api/models/Conversation.js
index 9f7aa90014..13c329aa4a 100644
--- a/api/models/Conversation.js
+++ b/api/models/Conversation.js
@@ -174,7 +174,7 @@ module.exports = {
if (search) {
try {
- const meiliResults = await Conversation.meiliSearch(search);
+ const meiliResults = await Conversation.meiliSearch(search, { filter: `user = "${user}"` });
const matchingIds = Array.isArray(meiliResults.hits)
? meiliResults.hits.map((result) => result.conversationId)
: [];
diff --git a/api/server/routes/messages.js b/api/server/routes/messages.js
index 0a277a1bd6..f1ac903ece 100644
--- a/api/server/routes/messages.js
+++ b/api/server/routes/messages.js
@@ -3,8 +3,8 @@ const { logger } = require('@librechat/data-schemas');
const { ContentTypes } = require('librechat-data-provider');
const {
saveConvo,
- saveMessage,
getMessage,
+ saveMessage,
getMessages,
updateMessage,
deleteMessages,
@@ -58,34 +58,51 @@ router.get('/', async (req, res) => {
const nextCursor = messages.length > pageSize ? messages.pop()[sortField] : null;
response = { messages, nextCursor };
} else if (search) {
- const searchResults = await Message.meiliSearch(search, undefined, true);
+ const searchResults = await Message.meiliSearch(search, { filter: `user = "${user}"` }, true);
const messages = searchResults.hits || [];
const result = await getConvosQueried(req.user.id, messages, cursor);
- const activeMessages = [];
+ const messageIds = [];
+ const cleanedMessages = [];
for (let i = 0; i < messages.length; i++) {
let message = messages[i];
if (message.conversationId.includes('--')) {
message.conversationId = cleanUpPrimaryKeyValue(message.conversationId);
}
if (result.convoMap[message.conversationId]) {
- const convo = result.convoMap[message.conversationId];
-
- const dbMessage = await getMessage({ user, messageId: message.messageId });
- activeMessages.push({
- ...message,
- title: convo.title,
- conversationId: message.conversationId,
- model: convo.model,
- isCreatedByUser: dbMessage?.isCreatedByUser,
- endpoint: dbMessage?.endpoint,
- iconURL: dbMessage?.iconURL,
- });
+ messageIds.push(message.messageId);
+ cleanedMessages.push(message);
}
}
+ const dbMessages = await getMessages({
+ user,
+ messageId: { $in: messageIds },
+ });
+
+ const dbMessageMap = {};
+ for (const dbMessage of dbMessages) {
+ dbMessageMap[dbMessage.messageId] = dbMessage;
+ }
+
+ const activeMessages = [];
+ for (const message of cleanedMessages) {
+ const convo = result.convoMap[message.conversationId];
+ const dbMessage = dbMessageMap[message.messageId];
+
+ activeMessages.push({
+ ...message,
+ title: convo.title,
+ conversationId: message.conversationId,
+ model: convo.model,
+ isCreatedByUser: dbMessage?.isCreatedByUser,
+ endpoint: dbMessage?.endpoint,
+ iconURL: dbMessage?.iconURL,
+ });
+ }
+
response = { messages: activeMessages, nextCursor: null };
} else {
response = { messages: [], nextCursor: null };
diff --git a/client/src/routes/Search.tsx b/client/src/routes/Search.tsx
index 4e6a180a17..a9f210832c 100644
--- a/client/src/routes/Search.tsx
+++ b/client/src/routes/Search.tsx
@@ -1,6 +1,5 @@
import { useEffect, useMemo } from 'react';
import { useRecoilValue } from 'recoil';
-import { buildTree } from 'librechat-data-provider';
import { Spinner, useToastContext } from '@librechat/client';
import MinimalMessagesWrapper from '~/components/Chat/Messages/MinimalMessages';
import { useNavScrolling, useLocalize, useAuthContext } from '~/hooks';
@@ -43,9 +42,20 @@ export default function Search() {
});
const messages = useMemo(() => {
- const msgs = searchMessages?.pages.flatMap((page) => page.messages) || [];
- const dataTree = buildTree({ messages: msgs, fileMap });
- return dataTree?.length === 0 ? null : (dataTree ?? null);
+ const msgs =
+ searchMessages?.pages.flatMap((page) =>
+ page.messages.map((message) => {
+ if (!message.files || !fileMap) {
+ return message;
+ }
+ return {
+ ...message,
+ files: message.files.map((file) => fileMap[file.file_id ?? ''] ?? file),
+ };
+ }),
+ ) || [];
+
+ return msgs.length === 0 ? null : msgs;
}, [fileMap, searchMessages?.pages]);
useEffect(() => {
diff --git a/packages/data-schemas/src/methods/share.test.ts b/packages/data-schemas/src/methods/share.test.ts
index 45b7faeb1a..1deec967e8 100644
--- a/packages/data-schemas/src/methods/share.test.ts
+++ b/packages/data-schemas/src/methods/share.test.ts
@@ -427,13 +427,14 @@ describe('Share Methods', () => {
expect(privateResults.links[0].title).toBe('Private Share');
});
- test('should handle search with mocked meiliSearch', async () => {
+ test('should handle search with mocked meiliSearch and user filter', async () => {
const userId = new mongoose.Types.ObjectId().toString();
// Mock meiliSearch method
- Conversation.meiliSearch = jest.fn().mockResolvedValue({
+ const meiliSearchMock = jest.fn().mockResolvedValue({
hits: [{ conversationId: 'conv1' }],
});
+ Conversation.meiliSearch = meiliSearchMock;
await SharedLink.create([
{
@@ -464,6 +465,9 @@ describe('Share Methods', () => {
expect(result.links).toHaveLength(1);
expect(result.links[0].title).toBe('Matching Share');
+
+ // Verify that meiliSearch was called with the correct user filter
+ expect(meiliSearchMock).toHaveBeenCalledWith('search term', { filter: `user = "${userId}"` });
});
test('should handle empty results', async () => {
@@ -475,6 +479,98 @@ describe('Share Methods', () => {
expect(result.nextCursor).toBeUndefined();
});
+ test('should only return shares from search results for the current user', async () => {
+ const userId1 = new mongoose.Types.ObjectId().toString();
+ const userId2 = new mongoose.Types.ObjectId().toString();
+
+ // Mock meiliSearch to simulate finding conversations from both users
+ const meiliSearchMock = jest.fn().mockImplementation((searchTerm, params) => {
+ // Simulate MeiliSearch filtering by user
+ const filter = params?.filter;
+ if (filter && filter.includes(userId1)) {
+ return Promise.resolve({
+ hits: [{ conversationId: 'conv1' }, { conversationId: 'conv3' }],
+ });
+ } else if (filter && filter.includes(userId2)) {
+ return Promise.resolve({ hits: [{ conversationId: 'conv2' }] });
+ }
+ // Without filter, would return all conversations (security issue)
+ return Promise.resolve({
+ hits: [
+ { conversationId: 'conv1' },
+ { conversationId: 'conv2' },
+ { conversationId: 'conv3' },
+ ],
+ });
+ });
+ Conversation.meiliSearch = meiliSearchMock;
+
+ // Create shares for different users
+ await SharedLink.create([
+ {
+ shareId: 'share1',
+ conversationId: 'conv1',
+ user: userId1,
+ title: 'User 1 Share',
+ isPublic: true,
+ },
+ {
+ shareId: 'share2',
+ conversationId: 'conv2',
+ user: userId2,
+ title: 'User 2 Share',
+ isPublic: true,
+ },
+ {
+ shareId: 'share3',
+ conversationId: 'conv3',
+ user: userId1,
+ title: 'Another User 1 Share',
+ isPublic: true,
+ },
+ ]);
+
+ // Search as userId1
+ const result1 = await shareMethods.getSharedLinks(
+ userId1,
+ undefined,
+ 10,
+ true,
+ 'createdAt',
+ 'desc',
+ 'search term',
+ );
+
+ // Should only get shares from conversations belonging to userId1
+ expect(result1.links).toHaveLength(2);
+ expect(result1.links.every((link) => link.title.includes('User 1'))).toBe(true);
+
+ // Verify correct filter was used
+ expect(meiliSearchMock).toHaveBeenCalledWith('search term', {
+ filter: `user = "${userId1}"`,
+ });
+
+ // Search as userId2
+ const result2 = await shareMethods.getSharedLinks(
+ userId2,
+ undefined,
+ 10,
+ true,
+ 'createdAt',
+ 'desc',
+ 'search term',
+ );
+
+ // Should only get shares from conversations belonging to userId2
+ expect(result2.links).toHaveLength(1);
+ expect(result2.links[0].title).toBe('User 2 Share');
+
+ // Verify correct filter was used for second user
+ expect(meiliSearchMock).toHaveBeenCalledWith('search term', {
+ filter: `user = "${userId2}"`,
+ });
+ });
+
test('should only return shares for the specified user', async () => {
const userId1 = new mongoose.Types.ObjectId().toString();
const userId2 = new mongoose.Types.ObjectId().toString();
diff --git a/packages/data-schemas/src/methods/share.ts b/packages/data-schemas/src/methods/share.ts
index 7c16ead209..8ff71fd718 100644
--- a/packages/data-schemas/src/methods/share.ts
+++ b/packages/data-schemas/src/methods/share.ts
@@ -150,7 +150,9 @@ export function createShareMethods(mongoose: typeof import('mongoose')) {
if (search && search.trim()) {
try {
- const searchResults = await Conversation.meiliSearch(search);
+ const searchResults = await Conversation.meiliSearch(search, {
+ filter: `user = "${user}"`,
+ });
if (!searchResults?.hits?.length) {
return {
diff --git a/packages/data-schemas/src/models/plugins/mongoMeili.ts b/packages/data-schemas/src/models/plugins/mongoMeili.ts
index c908135433..eacb3f2b60 100644
--- a/packages/data-schemas/src/models/plugins/mongoMeili.ts
+++ b/packages/data-schemas/src/models/plugins/mongoMeili.ts
@@ -1,6 +1,6 @@
import _ from 'lodash';
import { MeiliSearch } from 'meilisearch';
-import type { SearchResponse, Index } from 'meilisearch';
+import type { SearchResponse, SearchParams, Index } from 'meilisearch';
import type {
CallbackWithoutResultAndOptionalError,
FilterQuery,
@@ -75,7 +75,7 @@ export interface SchemaWithMeiliMethods extends Model<DocumentWithMeiliIndex> {
setMeiliIndexSettings(settings: Record<string, unknown>): Promise<unknown>;
meiliSearch(
q: string,
- params?: Record<string, unknown>,
+ params?: SearchParams,
populate?: boolean,
): Promise<SearchResponse<MeiliIndexable, Record<string, unknown>>>;
}
@@ -386,7 +386,7 @@ const createMeiliMongooseModel = ({
static async meiliSearch(
this: SchemaWithMeiliMethods,
q: string,
- params: Record<string, unknown>,
+ params: SearchParams,
populate: boolean,
): Promise<SearchResponse<MeiliIndexable, Record<string, unknown>>> {
const data = await index.search(q, params);
@@ -644,6 +644,16 @@ export default function mongoMeili(schema: Schema, options: MongoMeiliOptions):
logger.error(`[mongoMeili] Error checking index ${indexName}:`, error);
}
}
+
+ // Configure index settings to make 'user' field filterable
+ try {
+ await index.updateSettings({
+ filterableAttributes: ['user'],
+ });
+ logger.debug(`[mongoMeili] Updated index ${indexName} settings to make 'user' filterable`);
+ } catch (settingsError) {
+ logger.error(`[mongoMeili] Error updating index settings for ${indexName}:`, settingsError);
+ }
})();
// Collect attributes from the schema that should be indexed
@@ -654,6 +664,13 @@ export default function mongoMeili(schema: Schema, options: MongoMeiliOptions):
}, []),
];
+ // CRITICAL: Always include 'user' field for proper filtering
+ // This ensures existing deployments can filter by user after migration
+ if (schema.obj.user && !attributesToIndex.includes('user')) {
+ attributesToIndex.push('user');
+ logger.debug(`[mongoMeili] Added 'user' field to ${indexName} index attributes`);
+ }
+
schema.loadClass(createMeiliMongooseModel({ index, attributesToIndex, syncOptions }));
// Register Mongoose hooks
diff --git a/packages/data-schemas/src/schema/convo.ts b/packages/data-schemas/src/schema/convo.ts
index 9680a49d9e..4c3f09373f 100644
--- a/packages/data-schemas/src/schema/convo.ts
+++ b/packages/data-schemas/src/schema/convo.ts
@@ -19,6 +19,7 @@ const convoSchema: Schema<IConversation> = new Schema(
user: {
type: String,
index: true,
+ meiliIndex: true,
},
messages: [{ type: Schema.Types.ObjectId, ref: 'Message' }],
agentOptions: {
diff --git a/packages/data-schemas/src/schema/message.ts b/packages/data-schemas/src/schema/message.ts
index 15a80ae80e..c11252cb87 100644
--- a/packages/data-schemas/src/schema/message.ts
+++ b/packages/data-schemas/src/schema/message.ts
@@ -21,6 +21,7 @@ const messageSchema: Schema<IMessage> = new Schema(
index: true,
required: true,
default: null,
+ meiliIndex: true,
},
model: {
type: String,