Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🕳️ fix: Repair Legacy Meili Cleanup Index (#16133)

Danny Avilaf1079c050e22
diff --git a/packages/data-schemas/src/models/plugins/mongoMeili.spec.ts b/packages/data-schemas/src/models/plugins/mongoMeili.spec.ts
index ef97c98dcc..4c9a7511ca 100644
--- a/packages/data-schemas/src/models/plugins/mongoMeili.spec.ts
+++ b/packages/data-schemas/src/models/plugins/mongoMeili.spec.ts
@@ -1,8 +1,12 @@
import mongoose from 'mongoose';
import { EModelEndpoint } from 'librechat-data-provider';
import { MongoMemoryServer } from 'mongodb-memory-server';
+import type { Filter, Document as MongoDocument } from 'mongodb';
import type { SchemaWithMeiliMethods } from '~/models/plugins/mongoMeili';
-import mongoMeili, { MEILI_INDEX_SCHEMA_VERSION } from '~/models/plugins/mongoMeili';
+import mongoMeili, {
+ MEILI_INDEX_SCHEMA_VERSION,
+ buildExcludedIndexedQuery,
+} from '~/models/plugins/mongoMeili';
import { createConversationModel } from '~/models/convo';
import { createMessageModel } from '~/models/message';
import meiliLogger from '~/config/meiliLogger';
@@ -921,11 +925,11 @@ describe('Meilisearch Mongoose plugin', () => {
expect(conversationIndexes).toContainEqual([
{ _meiliIndex: 1, _meiliCleanupVersion: 1, conversationId: 1 },
expect.objectContaining({
- name: 'meili_excluded_legacy_cleanup_v3',
+ name: 'meili_excluded_legacy_cleanup_v4',
partialFilterExpression: {
subagentThread: { $exists: true },
_meiliIndex: { $eq: false },
- _meiliCleanupVersion: { $exists: false },
+ _meiliCleanupVersion: { $eq: null },
},
}),
]);
@@ -952,11 +956,11 @@ describe('Meilisearch Mongoose plugin', () => {
expect(messageIndexes).toContainEqual([
{ _meiliIndex: 1, _meiliCleanupVersion: 1, messageId: 1 },
expect.objectContaining({
- name: 'meili_excluded_legacy_cleanup_v3',
+ name: 'meili_excluded_legacy_cleanup_v4',
partialFilterExpression: {
subagentTask: { $exists: true },
_meiliIndex: { $eq: false },
- _meiliCleanupVersion: { $exists: false },
+ _meiliCleanupVersion: { $eq: null },
},
}),
]);
@@ -2430,4 +2434,115 @@ describe('Meilisearch Mongoose plugin', () => {
expect(afterSync.length).toBe(2);
});
});
+
+ describe('excluded-document cleanup indexes', () => {
+ test('builds every cleanup index the plugin declares', async () => {
+ const messageModel = createMessageModel(mongoose);
+
+ await expect(messageModel.createIndexes()).resolves.toBeUndefined();
+
+ const builtIndexes = await messageModel.collection.listIndexes().toArray();
+ expect(builtIndexes.map((builtIndex) => builtIndex.name)).toEqual(
+ expect.arrayContaining([
+ 'meili_excluded_indexed_cleanup_v3',
+ 'meili_excluded_attempted_cleanup_v3',
+ 'meili_excluded_legacy_cleanup_v4',
+ ]),
+ );
+
+ const legacyCleanup = builtIndexes.find(
+ (builtIndex) => builtIndex.name === 'meili_excluded_legacy_cleanup_v4',
+ );
+ expect(legacyCleanup?.key).toEqual({
+ _meiliIndex: 1,
+ _meiliCleanupVersion: 1,
+ messageId: 1,
+ });
+ /** `$exists: false` is the operator MongoDB rejects in a partial filter, so the
+ * unstamped state is expressed as a null equality and must stay that way. */
+ expect(legacyCleanup?.partialFilterExpression).toEqual({
+ subagentTask: { $exists: true },
+ _meiliIndex: { $eq: false },
+ _meiliCleanupVersion: { $eq: null },
+ });
+ });
+
+ test('cleans up an excluded document stored before the cleanup version existed', async () => {
+ const messageModel = createMessageModel(mongoose) as unknown as SchemaWithMeiliMethods;
+ const messageId = new mongoose.Types.ObjectId().toString();
+ await messageModel.collection.insertOne({
+ messageId,
+ conversationId: new mongoose.Types.ObjectId().toString(),
+ user: 'user-legacy-cleanup',
+ isCreatedByUser: true,
+ text: 'Indexed before the private child marker existed',
+ subagentTask: { attemptKey: 'attempt-key', status: 'completed' },
+ _meiliIndex: false,
+ });
+
+ await messageModel.cleanupExcludedMeiliIndex();
+
+ expect(mockDeleteDocuments).toHaveBeenCalledWith(expect.arrayContaining([messageId]));
+ const cleaned = await messageModel.collection.findOne({ messageId });
+ expect(cleaned?._meiliCleanupVersion).toBe(1);
+ expect(cleaned?._meiliIndex).toBeUndefined();
+ });
+
+ test('holds only excluded documents still awaiting cleanup', async () => {
+ const messageModel = createMessageModel(mongoose);
+ await messageModel.createIndexes();
+ const conversationId = new mongoose.Types.ObjectId().toString();
+ const awaitingCleanup = new mongoose.Types.ObjectId().toString();
+ const alreadyStamped = new mongoose.Types.ObjectId().toString();
+ await messageModel.collection.insertMany([
+ {
+ messageId: awaitingCleanup,
+ conversationId,
+ user: 'user-index-population',
+ isCreatedByUser: true,
+ text: 'Written before the cleanup version existed',
+ subagentTask: { attemptKey: 'attempt-key', status: 'completed' },
+ _meiliIndex: false,
+ },
+ {
+ messageId: alreadyStamped,
+ conversationId,
+ user: 'user-index-population',
+ isCreatedByUser: true,
+ text: 'Written by the current schema, nothing to clean up',
+ subagentTask: { attemptKey: 'attempt-key', status: 'completed' },
+ _meiliIndex: false,
+ _meiliCleanupVersion: 1,
+ },
+ ]);
+
+ /** Hinting the partial index reaches only the documents it holds, which is the
+ * invariant that keeps it from growing with every private subagent document. */
+ const indexedMessageIds = await messageModel.collection
+ .find({ user: 'user-index-population' })
+ .hint('meili_excluded_legacy_cleanup_v4')
+ .project({ messageId: 1, _id: 0 })
+ .toArray();
+
+ expect(indexedMessageIds.map((document) => document.messageId)).toEqual([awaitingCleanup]);
+ });
+
+ test('serves the legacy cleanup branch from the index', async () => {
+ const messageModel = createMessageModel(mongoose);
+ await messageModel.createIndexes();
+ const excludedQuery = buildExcludedIndexedQuery('subagentTask');
+ const branches = (excludedQuery?.$or ?? []) as Filter<MongoDocument>[];
+ const legacyBranch = branches.find((branch) => branch._meiliIndex === false);
+ const legacyFilter: Filter<MongoDocument> = {
+ subagentTask: { $exists: true },
+ ...legacyBranch,
+ };
+
+ const plan = await messageModel.collection.find(legacyFilter).explain('queryPlanner');
+
+ expect(JSON.stringify(plan.queryPlanner.winningPlan)).toContain(
+ 'meili_excluded_legacy_cleanup_v4',
+ );
+ });
+ });
});
diff --git a/packages/data-schemas/src/models/plugins/mongoMeili.ts b/packages/data-schemas/src/models/plugins/mongoMeili.ts
index a5c307ea6e..2adfda4444 100644
--- a/packages/data-schemas/src/models/plugins/mongoMeili.ts
+++ b/packages/data-schemas/src/models/plugins/mongoMeili.ts
@@ -186,7 +186,16 @@ const buildIndexableQuery = (
};
};
-const buildExcludedIndexedQuery = (excludeFromIndexPath?: string): FilterQuery<unknown> | null => {
+/**
+ * Excluded documents that may still hold a Meili entry. The legacy branch matches a
+ * `_meiliCleanupVersion` that is absent or null, written as a null equality rather than
+ * `$exists: false` so `meili_excluded_legacy_cleanup_v4` can serve it: a partial index
+ * accepts null equality and rejects `$exists: false`, and the planner only reaches a
+ * partial index through a predicate that implies its filter.
+ */
+export const buildExcludedIndexedQuery = (
+ excludeFromIndexPath?: string,
+): FilterQuery<unknown> | null => {
if (excludeFromIndexPath == null) {
return null;
}
@@ -196,7 +205,7 @@ const buildExcludedIndexedQuery = (excludeFromIndexPath?: string): FilterQuery<u
$or: [
{ _meiliIndex: true },
{ _meiliIndexAttempted: true },
- { _meiliIndex: false, _meiliCleanupVersion: { $exists: false } },
+ { _meiliIndex: false, _meiliCleanupVersion: null },
],
};
};
@@ -988,14 +997,21 @@ export default function mongoMeili(schema: Schema, options: MongoMeiliOptions):
},
},
);
+ /* Serves the legacy branch of `buildExcludedIndexedQuery`. MongoDB rewrites
+ * `$exists: false` into `$not`, which no `partialFilterExpression` accepts, so the
+ * unstamped state is expressed as a null equality: it admits an absent or null
+ * `_meiliCleanupVersion` and keeps every already-stamped document out, which is what
+ * bounds this index to the shrinking legacy population rather than to every
+ * excluded document ever written. Cleanup stamps the version, and the document
+ * leaves the index. */
schema.index(
{ _meiliIndex: 1, _meiliCleanupVersion: 1, [options.primaryKey]: 1 },
{
- name: 'meili_excluded_legacy_cleanup_v3',
+ name: 'meili_excluded_legacy_cleanup_v4',
partialFilterExpression: {
[options.excludeFromIndexPath]: { $exists: true },
_meiliIndex: { $eq: false },
- _meiliCleanupVersion: { $exists: false },
+ _meiliCleanupVersion: { $eq: null },
},
},
);