Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🍃 fix: Serialize Index Builds and the Agent Edge Cleanup for DocumentDB (#15670)
Danny Avila84851d86553b
diff --git a/packages/api/src/agents/checkpointer.integration.spec.ts b/packages/api/src/agents/checkpointer.integration.spec.tsindex a992822485..5a490a8a3e 100644--- a/packages/api/src/agents/checkpointer.integration.spec.ts+++ b/packages/api/src/agents/checkpointer.integration.spec.ts@@ -16,6 +16,7 @@ import {CheckpointTooLargeError,LIBRECHAT_CHECKPOINT_NAMESPACE_KEY,LIBRECHAT_EVENT_ACTOR_INVOCATION_KEY,+ setupCheckpointIndexes,__resetCheckpointerForTests,} from './checkpointer';@@ -1135,3 +1136,230 @@ describe('LazyMongoSaver checkpoint size guard (mongodb-memory-server integratioexpect(channels).toContain(NO_WRITES); // flushed, not dropped});});++describe('setupCheckpointIndexes on a single-index-build engine (mongodb-memory-server)', () => {+ const CHECKPOINTS = 'single_build_checkpoints';+ const WRITES = 'single_build_checkpoint_writes';+ /** Amazon DocumentDB: "Existing index build in progress on the same collection." */+ const INDEX_BUILD_ALREADY_IN_PROGRESS = 40333;++ const clientForSaver = () =>+ mongoose.connection.getClient() as unknown as ConstructorParameters<+ typeof MongoDBSaver+ >[0]['client'];++ /** Bound to Mongoose's database the way `buildMongoSaver` binds it, so the+ * index assertions below read the collections the saver actually built. */+ const makeSaver = () =>+ new LazyMongoSaver({+ client: clientForSaver(),+ dbName: mongoose.connection.db?.databaseName,+ checkpointCollectionName: CHECKPOINTS,+ checkpointWritesCollectionName: WRITES,+ ttl: 3600,+ });++ function indexBuildInProgressError(): Error {+ return new mongoose.mongo.MongoServerError({+ ok: 0,+ code: INDEX_BUILD_ALREADY_IN_PROGRESS,+ errmsg:+ 'Existing index build in progress on the same collection. Collection is limited to a single index build at a time.',+ });+ }++ interface IndexBuild {+ collectionName: string;+ indexName: string;+ build: () => Promise<string>;+ }++ /**+ * Routes every `createIndex` through `handler`, naming the index the way the+ * server would. Patched on the driver prototype because the saver fetches a+ * fresh `Collection` handle per call.+ */+ function patchCreateIndex(handler: (build: IndexBuild) => Promise<string>): () => void {+ const prototype = mongoose.mongo.Collection.prototype;+ const createIndex = prototype.createIndex;+ prototype.createIndex = function (this: mongoose.mongo.Collection, spec, options) {+ const indexName =+ options?.name ??+ Object.entries(spec)+ .map(([field, direction]) => `${field}_${direction}`)+ .join('_');+ return handler({+ collectionName: this.collectionName,+ indexName,+ build: () => createIndex.call(this, spec, options),+ });+ };+ return () => {+ prototype.createIndex = createIndex;+ };+ }++ /**+ * Turns the in-memory MongoDB into a single-index-build engine for the two+ * checkpoint collections: a build that arrives while another is in flight on+ * the same collection is rejected the way DocumentDB rejects it, instead of+ * being serialized the way MongoDB does. Re-creating an index that already+ * exists starts no build on either engine, so it always passes through.+ */+ function enforceSingleIndexBuild(): { rejected: () => number; restore: () => void } {+ const inFlight = new Set<string>();+ const built = new Set<string>();+ let rejected = 0;+ const restore = patchCreateIndex(async ({ collectionName, indexName, build }) => {+ const key = `${collectionName}:${indexName}`;+ if ((collectionName !== CHECKPOINTS && collectionName !== WRITES) || built.has(key)) {+ return build();+ }+ if (inFlight.has(collectionName)) {+ rejected += 1;+ throw indexBuildInProgressError();+ }+ inFlight.add(collectionName);+ try {+ await new Promise((resolve) => setTimeout(resolve, 5));+ const name = await build();+ built.add(key);+ return name;+ } finally {+ inFlight.delete(collectionName);+ }+ });+ return { rejected: () => rejected, restore };+ }++ async function indexNames(collectionName: string): Promise<string[]> {+ const indexes = await mongoose.connection.db!.collection(collectionName).indexes();+ return indexes.map((index) => index.name ?? '');+ }++ it('models the engine: a raw setup() loses one build per collection', async () => {+ const engine = enforceSingleIndexBuild();+ try {+ const errors = await makeSaver().setup();++ expect(errors.map((error) => (error as { code?: number }).code)).toEqual([+ INDEX_BUILD_ALREADY_IN_PROGRESS,+ INDEX_BUILD_ALREADY_IN_PROGRESS,+ ]);+ expect(engine.rejected()).toBe(2);+ } finally {+ engine.restore();+ }+ });++ it('re-runs setup() until every checkpoint index exists', async () => {+ const engine = enforceSingleIndexBuild();+ try {+ await expect(+ setupCheckpointIndexes(makeSaver(), { peerBuildPollMs: 1, peerBuildDeadlineMs: 5_000 }),+ ).resolves.toEqual([]);++ expect(engine.rejected()).toBe(2);+ expect(await indexNames(CHECKPOINTS)).toEqual(+ expect.arrayContaining(['thread_ns_checkpoint_idx', 'upserted_at_1']),+ );+ expect(await indexNames(WRITES)).toEqual(+ expect.arrayContaining(['thread_ns_checkpoint_task_idx', 'upserted_at_1']),+ );+ } finally {+ engine.restore();+ }+ });++ it('returns errors that are not a concurrent build without re-running setup()', async () => {+ const restore = patchCreateIndex(async ({ collectionName, build }) => {+ if (collectionName !== WRITES) {+ return build();+ }+ throw new mongoose.mongo.MongoServerError({+ ok: 0,+ code: 67,+ errmsg: 'CannotCreateIndex: bad index spec',+ });+ });+ try {+ const saver = makeSaver();+ const setup = jest.spyOn(saver, 'setup');++ const errors = await setupCheckpointIndexes(saver, {+ peerBuildPollMs: 1,+ peerBuildDeadlineMs: 5_000,+ });++ expect(errors.map((error) => error.message)).toEqual([+ 'CannotCreateIndex: bad index spec',+ 'CannotCreateIndex: bad index spec',+ ]);+ expect(setup).toHaveBeenCalledTimes(1);+ } finally {+ restore();+ }+ });++ it('proceeds with the conflict reported once the peer-build deadline passes', async () => {+ const restore = patchCreateIndex(async ({ collectionName, build }) => {+ if (collectionName !== CHECKPOINTS) {+ return build();+ }+ throw indexBuildInProgressError();+ });+ try {+ const errors = await setupCheckpointIndexes(makeSaver(), {+ peerBuildPollMs: 1,+ peerBuildDeadlineMs: 20,+ });++ expect(errors.map((error) => (error as { code?: number }).code)).toEqual([+ INDEX_BUILD_ALREADY_IN_PROGRESS,+ ]);+ } finally {+ restore();+ }+ });++ it('keeps the failures reported beside a conflict that outlasts the deadline', async () => {+ const restore = patchCreateIndex(async ({ collectionName }) => {+ if (collectionName === CHECKPOINTS) {+ throw indexBuildInProgressError();+ }+ throw new mongoose.mongo.MongoServerError({+ ok: 0,+ code: 67,+ errmsg: 'CannotCreateIndex: bad index spec',+ });+ });+ try {+ const errors = await setupCheckpointIndexes(makeSaver(), {+ peerBuildPollMs: 1,+ peerBuildDeadlineMs: 20,+ });++ expect(errors.map((error) => (error as { code?: number }).code).sort()).toEqual([+ INDEX_BUILD_ALREADY_IN_PROGRESS,+ 67,+ 67,+ ]);+ } finally {+ restore();+ }+ });++ it('propagates a setup() rejection that is not a build conflict', async () => {+ /** The driver validates collection names server-side and `setup()` settles+ * every build with `Promise.allSettled`, so there is no rejection path to+ * trigger for real; a rejecting stand-in covers the contract that such a+ * rejection reaches the caller's in-process fallback instead of being+ * reported as an index error. */+ const saver = makeSaver();+ saver.setup = () => Promise.reject(new Error('client closed'));++ await expect(setupCheckpointIndexes(saver, { peerBuildPollMs: 1 })).rejects.toThrow(+ 'client closed',+ );+ });+});diff --git a/packages/api/src/agents/checkpointer.ts b/packages/api/src/agents/checkpointer.tsindex 63005be922..055867eaf1 100644--- a/packages/api/src/agents/checkpointer.ts+++ b/packages/api/src/agents/checkpointer.ts@@ -1,7 +1,7 @@import mongoose from 'mongoose';-import { logger } from '@librechat/data-schemas';import { INTERRUPT } from '@langchain/langgraph-checkpoint';import { MongoDBSaver } from '@langchain/langgraph-checkpoint-mongodb';+import { buildIndexWithRetry, isIndexBuildInProgress, logger } from '@librechat/data-schemas';import type {Checkpoint,CheckpointListOptions,@@ -11,6 +11,7 @@ import type {} from '@langchain/langgraph-checkpoint';import type { BaseMessage } from '@librechat/agents/langchain/messages';import type { TCheckpointerConfig } from 'librechat-data-provider';+import type { IndexBuildOptions } from '@librechat/data-schemas';import type { RunnableConfig } from '@langchain/core/runnables';/**@@ -805,6 +806,48 @@ export async function captureAgentEventCheckpoint(: null;}+/** Longest wait for a peer's checkpoint index build before the saver proceeds without it. */+const CHECKPOINT_INDEX_BUILD_DEADLINE_MS = 120_000;++/**+ * `MongoDBSaver.setup()` starts the compound and TTL builds of each collection+ * concurrently and reports failures instead of throwing. Amazon DocumentDB+ * admits one index build per collection at a time, so there the second build of+ * each pair is rejected (code 40333) and the saver would otherwise run with its+ * TTL index missing — checkpoints then never expire. `setup()` is idempotent (an+ * index that already exists is a no-op), so re-running it lets one more build+ * through per pass until every index exists. Every other error is returned for+ * the caller to log, exactly as `setup()` reports it — including those reported+ * beside a conflict that outlasts the deadline. A rejection of `setup()` itself+ * propagates, so the caller keeps its in-process fallback.+ */+export async function setupCheckpointIndexes(+ saver: Pick<MongoDBSaver, 'setup'>,+ options: IndexBuildOptions = {},+): Promise<Error[]> {+ let companions: Error[] = [];+ try {+ return await buildIndexWithRetry(+ async () => {+ const errors = await saver.setup();+ const blocked = errors.find(isIndexBuildInProgress);+ if (blocked == null) {+ return errors;+ }+ companions = errors.filter((error) => !isIndexBuildInProgress(error));+ throw blocked;+ },+ 'MongoDBSaver.setup()',+ { peerBuildDeadlineMs: CHECKPOINT_INDEX_BUILD_DEADLINE_MS, ...options },+ );+ } catch (error) {+ if (!isIndexBuildInProgress(error)) {+ throw error;+ }+ return [...companions, error instanceof Error ? error : new Error(String(error))];+ }+}+async function buildMongoSaver(resolved: ResolvedCheckpointerConfig,): Promise<MongoDBSaver | undefined> {@@ -829,7 +872,7 @@ async function buildMongoSaver(// approval window, so a forgotten approval can never leak checkpoints forever.ttl: resolved.ttlSeconds,});- const errors = await saver.setup();+ const errors = await setupCheckpointIndexes(saver);if (errors.length > 0) {logger.warn('[checkpointer] MongoDBSaver.setup() reported errors (checkpoint indexes may be incomplete):',diff --git a/packages/data-schemas/misc/documentdb/documentdb-compat.md b/packages/data-schemas/misc/documentdb/documentdb-compat.mdindex 8152e360c2..a86d38a212 100644--- a/packages/data-schemas/misc/documentdb/documentdb-compat.md+++ b/packages/data-schemas/misc/documentdb/documentdb-compat.md@@ -161,18 +161,69 @@ in the docs.- **DocumentDB 8.0 pipeline-update acceptance** — 8.0 added `$set`/`$unset`aggregation _stages_, but AWS never documents pipeline-form updates; theharness probe answers this live.-- **`collMod` is only "Partial"** on every version — avoid- `Model.syncIndexes()` against DocumentDB (it may issue `collMod` beyond the- documented `expireAfterSeconds`).-- **Read-side aggregations** (3 files: `methods/prompt.ts`,- `methods/aclEntry.ts`, `methods/agentCategory.ts`) were not audited- stage-by-stage; no exotic stages (`$facet`, `$setWindowFields`,- `$unionWith`, `$graphLookup`) are used anywhere.+- **`collMod` is only "Partial"** on every version — `Model.syncIndexes()`+ may issue `collMod` beyond the documented `expireAfterSeconds`, so the static+ guard rejects it outright.+- **Aggregation operators — now enforced, not observed.** 29 `aggregate`+ calls across 11 files (`insights.ts`, `message.ts`, `mcpAuthority.ts`,+ `agent.ts`, `aclEntry.ts`, `triggerDelivery.ts`, `queuedTurn.ts`,+ `prompt.ts`, `conversation.ts`, `agentCategory.ts`,+ `packages/api/src/code/lifecycle.ts`). An earlier inventory here named three+ files because it was built from `.aggregate(`, which never matches the+ generically typed `.aggregate<T>(` form the newer code uses. Every operator+ in use (`$strLenBytes`, `$let`, `$map`, `$reduce`, `$regexMatch`, `$convert`,+ `$anyElementTrue`, …) is in AWS's 5.0 supported list, and+ `methods/documentdb.spec.ts` now rejects the full unsupported matrix,+ `$set`/`$unset` at stage position, and the `$count` accumulator. Several+ operators are 4.0+/5.0-only (`$expr`, `$switch`, `$convert`, `$regexMatch`,+ array-form `$first`), which reinforces the 5.0+ recommendation.- **No faithful local emulator exists.** The `documentdb-local` Docker imageis the PostgreSQL-based Linux Foundation project — AWS's own OSS blogconfirms "a different engine than the one used in Amazon DocumentDB." Liveregression testing must run against a real cluster.+## Proven, fixed — edge cleanup pipeline update (every version)++`removeAgentIdsFromEdges` (`methods/agent.ts`, since #14428) pruned deleted+agent ids out of every graph's `edges` with an aggregation-pipeline update —+the class fixed everywhere else in #15375. The static guard missed it because+the pipeline reached `updateMany` as a function's return value rather than an+array literal or an array-bound variable; the guard now follows calls to+functions declared or annotated to return an array, and independently rejects+`$set`/`$unset` at stage position. The rewrite reads each matching graph,+prunes in code, and writes the result back behind a compare-and-set on the+edges it read (`tenantSafeBulkWrite`, one round trip for every graph), so a+concurrent edit is never overwritten — the atomicity the pipeline provided, at+the cost of one extra read on the agent-delete path.++## Proven, fixed — concurrent index builds (every version)++DocumentDB admits one index build per collection at a time and rejects a+second with code 40333 where MongoDB would serialize it. `utils/retry.ts`+exists for exactly this and the boot builds go through it — but three callers+bypassed it, and one sits on the agent hot path:++- **`MongoDBSaver.setup()`** (`@langchain/langgraph-checkpoint-mongodb`) starts+ the compound and TTL builds of each checkpoint collection in a single+ `Promise.allSettled` and returns the rejections instead of throwing;+ `buildMongoSaver` logged them and proceeded. On DocumentDB one build per+ collection lost that race on every boot — the TTL index is pushed second,+ making it the likely loser, so checkpoints accumulated without bound.+ `setupCheckpointIndexes` now re-runs the idempotent `setup()` while any+ rejection is a 40333; each pass admits one more build until all exist.+- `methods/auditLog.ts` called `model.createIndexes()` raw before the chain's+ first append; `migrations/mcpAuthorityIndexes.ts` and+ `migrations/mcpServerNames.ts` called `collection.createIndex()` raw. All+ three now go through the helpers (`buildIndexWithRetry` is the+ raw-collection form of `createIndexesWithRetry`).++`methods/documentdb.spec.ts` rejects any `createIndex` / `createIndexes` /+`syncIndexes` / `ensureIndexes` outside `utils/retry.ts` unless it is the+argument of `buildIndexWithRetry`, so the class cannot recur silently. The+method sweep cannot see this class at all — it drives exported methods, and+these builds happen inside a third-party `setup()` and in migrations — which+is how it survived three audits.+## Regression strategy1. **In-repo (CI today)**: the behavioral tests added in@@ -189,14 +240,15 @@ in the docs.## Support matrix and recommendation-| Capability (LibreChat dependency) | 3.6 | 4.0 | 5.0 | 8.0 | Elastic |-| --------------------------------------- | --- | --- | --- | --- | ------- |-| Pipeline updates (**no longer used**) | ✗ | ✗ | ✗ | ? | ✗ |-| Plain update operators (all writes now) | ✓ | ✓ | ✓ | ✓ | ✓ |-| Unique indexes | ✓ | ✓ | ✓ | ✓ | ✗ |-| Partial unique indexes (OAuth ids) | ✗ | ✗ | ✓ | ✓ | ✗ |-| Transactions (runtime-probed) | ✗ | ✓ | ✓ | ✓ | ✗ |-| TTL indexes | ✓ | ✓ | ✓ | ✓ | ✓ |+| Capability (LibreChat dependency) | 3.6 | 4.0 | 5.0 | 8.0 | Elastic |+| ---------------------------------------- | --- | --- | --- | --- | ------- |+| Pipeline updates (**no longer used**) | ✗ | ✗ | ✗ | ? | ✗ |+| Plain update operators (all writes now) | ✓ | ✓ | ✓ | ✓ | ✓ |+| Unique indexes | ✓ | ✓ | ✓ | ✓ | ✗ |+| Partial unique indexes (OAuth ids) | ✗ | ✗ | ✓ | ✓ | ✗ |+| Transactions (runtime-probed) | ✗ | ✓ | ✓ | ✓ | ✗ |+| TTL indexes | ✓ | ✓ | ✓ | ✓ | ✓ |+| Concurrent index builds (now serialized) | ✗ | ✗ | ✗ | ✗ | ✗ |**Recommendation**: support **DocumentDB 5.0+ instance-based** with`retryWrites=false` documented as required. 4.0 functions withdiff --git a/packages/data-schemas/src/methods/agent.spec.ts b/packages/data-schemas/src/methods/agent.spec.tsindex a371438ea1..be07f8f2e9 100644--- a/packages/data-schemas/src/methods/agent.spec.ts+++ b/packages/data-schemas/src/methods/agent.spec.ts@@ -20,8 +20,13 @@ import type {Model,} from 'mongoose';import type { IAgent, IAclEntry, IUser, IAccessRole, CodeEnvironmentDocument } from '..';+import {+ createAgentMethods,+ EDGE_CLEANUP_BATCH,+ EDGE_CLEANUP_MAX_SWEEPS,+ type AgentMethods,+} from './agent';import { withCodeEnvironmentReference } from './codeEnvironment';-import { createAgentMethods, type AgentMethods } from './agent';import { tenantStorage } from '~/config/tenantContext';import { createAclEntryMethods } from './aclEntry';import { createModels } from '~/models';@@ -1265,6 +1270,207 @@ describe('Agent Methods', () => {]);});+ test('keeps an edge added to the graph while the cleanup was running', async () => {+ const authorId = new mongoose.Types.ObjectId();+ const deletedAgentId = `agent_${uuidv4()}`;+ const graphAgentId = `agent_${uuidv4()}`;+ const sourceAgentId = `agent_${uuidv4()}`;+ const targetAgentId = `agent_${uuidv4()}`;+ const addedAgentId = `agent_${uuidv4()}`;+ await createAgent({+ id: deletedAgentId,+ name: 'Agent To Delete',+ provider: 'test',+ model: 'test-model',+ author: authorId,+ });+ await createAgent({+ id: graphAgentId,+ name: 'Agent Edited During Cleanup',+ provider: 'test',+ model: 'test-model',+ author: authorId,+ edges: [+ { from: deletedAgentId, to: targetAgentId, edgeType: 'handoff' },+ { from: sourceAgentId, to: targetAgentId, edgeType: 'handoff' },+ ],+ });+ /** Lands a concurrent edit between the cleanup's read and its first write,+ * through the driver so the cleanup's own compare-and-set is what must+ * notice it. */+ const prototype = mongoose.mongo.Collection.prototype;+ const bulkWrite = prototype.bulkWrite;+ let edited = false;+ prototype.bulkWrite = async function (this: mongoose.mongo.Collection, operations, options) {+ if (!edited && this.collectionName === 'agents') {+ edited = true;+ await Agent.updateOne(+ { id: graphAgentId },+ { $push: { edges: { from: sourceAgentId, to: addedAgentId, edgeType: 'handoff' } } },+ );+ }+ return bulkWrite.call(this, operations, options);+ };++ try {+ await deleteAgent({ id: deletedAgentId });+ } finally {+ prototype.bulkWrite = bulkWrite;+ }++ expect(edited).toBe(true);+ const graphAgent = await getAgent({ id: graphAgentId });+ expect(graphAgent!.edges).toEqual([+ { from: sourceAgentId, to: targetAgentId, edgeType: 'handoff' },+ { from: sourceAgentId, to: addedAgentId, edgeType: 'handoff' },+ ]);+ });++ test('cleans every graph when the references span more than one page', async () => {+ const authorId = new mongoose.Types.ObjectId();+ const deletedAgentId = `agent_${uuidv4()}`;+ const targetAgentId = `agent_${uuidv4()}`;+ const graphCount = EDGE_CLEANUP_BATCH + 1;+ await createAgent({+ id: deletedAgentId,+ name: 'Popular Agent',+ provider: 'test',+ model: 'test-model',+ author: authorId,+ });+ await Agent.insertMany(+ Array.from({ length: graphCount }, (_, index) => ({+ id: `agent_${uuidv4()}`,+ name: `Paged Graph ${index}`,+ provider: 'test',+ model: 'test-model',+ author: authorId,+ edges: [+ { from: deletedAgentId, to: targetAgentId, edgeType: 'handoff' },+ { from: targetAgentId, to: [deletedAgentId, targetAgentId], edgeType: 'direct' },+ ],+ })),+ );++ await deleteAgent({ id: deletedAgentId });++ const graphs = await Agent.find({ name: /^Paged Graph / })+ .select('edges')+ .lean<Pick<IAgent, 'edges'>[]>();+ expect(graphs).toHaveLength(graphCount);+ const expectedEdges = [{ from: targetAgentId, to: [targetAgentId], edgeType: 'direct' }];+ graphs.forEach((graph) => expect(graph.edges).toEqual(expectedEdges));+ });++ test('retries a compare-and-set miss the cursor has already passed', async () => {+ const authorId = new mongoose.Types.ObjectId();+ const deletedAgentId = `agent_${uuidv4()}`;+ const targetAgentId = `agent_${uuidv4()}`;+ const addedAgentId = `agent_${uuidv4()}`;+ const graphCount = EDGE_CLEANUP_BATCH + 1;+ await createAgent({+ id: deletedAgentId,+ name: 'Popular Agent',+ provider: 'test',+ model: 'test-model',+ author: authorId,+ });+ const graphIds = Array.from({ length: graphCount }, () => `agent_${uuidv4()}`);+ await Agent.insertMany(+ graphIds.map((id, index) => ({+ id,+ name: `Cursor Graph ${index}`,+ provider: 'test',+ model: 'test-model',+ author: authorId,+ edges: [+ { from: deletedAgentId, to: [deletedAgentId, targetAgentId], edgeType: 'direct' },+ ],+ })),+ );+ const [editedGraphId] = graphIds;+ /** Lands a concurrent edit on a first-page graph between the cleanup's read+ * and its first write, so that graph's compare-and-set misses while the+ * cursor moves on past it. */+ const prototype = mongoose.mongo.Collection.prototype;+ const bulkWrite = prototype.bulkWrite;+ let edited = false;+ prototype.bulkWrite = async function (this: mongoose.mongo.Collection, operations, options) {+ if (!edited && this.collectionName === 'agents') {+ edited = true;+ await Agent.updateOne(+ { id: editedGraphId },+ { $push: { edges: { from: targetAgentId, to: addedAgentId, edgeType: 'handoff' } } },+ );+ }+ return bulkWrite.call(this, operations, options);+ };++ try {+ await deleteAgent({ id: deletedAgentId });+ } finally {+ prototype.bulkWrite = bulkWrite;+ }++ expect(edited).toBe(true);+ expect(+ await Agent.countDocuments({+ $or: [{ 'edges.from': deletedAgentId }, { 'edges.to': deletedAgentId }],+ }),+ ).toBe(0);+ const editedGraph = await getAgent({ id: editedGraphId });+ expect(editedGraph!.edges).toEqual([+ { from: targetAgentId, to: addedAgentId, edgeType: 'handoff' },+ ]);+ });++ test('gives up after a bounded number of sweeps when references keep being added', async () => {+ const authorId = new mongoose.Types.ObjectId();+ const deletedAgentId = `agent_${uuidv4()}`;+ const graphAgentId = `agent_${uuidv4()}`;+ const targetAgentId = `agent_${uuidv4()}`;+ await createAgent({+ id: deletedAgentId,+ name: 'Agent To Delete',+ provider: 'test',+ model: 'test-model',+ author: authorId,+ });+ await createAgent({+ id: graphAgentId,+ name: 'Graph That Keeps Referencing',+ provider: 'test',+ model: 'test-model',+ author: authorId,+ edges: [{ from: deletedAgentId, to: targetAgentId, edgeType: 'handoff' }],+ });+ /** Re-adds a reference AFTER every successful write, so every sweep makes+ * progress and the stall bound never trips; only the sweep bound ends it. */+ const prototype = mongoose.mongo.Collection.prototype;+ const bulkWrite = prototype.bulkWrite;+ let writes = 0;+ prototype.bulkWrite = async function (this: mongoose.mongo.Collection, operations, options) {+ const result = await bulkWrite.call(this, operations, options);+ if (this.collectionName === 'agents') {+ writes += 1;+ await Agent.updateOne(+ { id: graphAgentId },+ { $push: { edges: { from: deletedAgentId, to: targetAgentId, edgeType: 'handoff' } } },+ );+ }+ return result;+ };++ try {+ await deleteAgent({ id: deletedAgentId });+ } finally {+ prototype.bulkWrite = bulkWrite;+ }++ expect(await getAgent({ id: deletedAgentId })).toBeNull();+ expect(writes).toBeLessThanOrEqual(EDGE_CLEANUP_MAX_SWEEPS);+ });+test('should remove agent from user favorites when agent is deleted', async () => {const agentId = `agent_${uuidv4()}`;const authorId = new mongoose.Types.ObjectId();diff --git a/packages/data-schemas/src/methods/agent.ts b/packages/data-schemas/src/methods/agent.tsindex 4f2d392221..a11ba807ad 100644--- a/packages/data-schemas/src/methods/agent.ts+++ b/packages/data-schemas/src/methods/agent.ts@@ -7,10 +7,11 @@ import {actionDelimiter,isActionTool,} from 'librechat-data-provider';-import type { FilterQuery, Model, PipelineStage, ProjectionType, Types } from 'mongoose';+import type { FilterQuery, Model, ProjectionType, Types } from 'mongoose';import type { AgentToolResources } from 'librechat-data-provider';import type { IAgent, IAclEntry, ActionQuery } from '~/types';import { withCodeEnvironmentReference } from './codeEnvironment';+import { tenantSafeBulkWrite } from '~/utils/tenantBulkWrite';import { filterExistingSkillIds } from './skill';import logger from '~/config/winston';@@ -45,71 +46,57 @@ const TOOL_RESOURCE_KEYS: ReadonlyArray<keyof AgentToolResources> = [EToolResources.ocr,];-/** Builds an atomic update that prunes deleted IDs without discarding surviving edge members. */-function createEdgeCleanupPipeline(agentIds: string[]): PipelineStage[] {- const cleanEndpoint = (endpoint: string) => ({- $cond: [- { $isArray: endpoint },- {- $filter: {- input: endpoint,- as: 'agentId',- cond: { $not: [{ $in: ['$$agentId', agentIds] }] },- },- },- { $cond: [{ $in: [endpoint, agentIds] }, null, endpoint] },- ],- });- const hasEndpoint = (endpoint: string) => ({- $cond: [{ $isArray: endpoint }, { $gt: [{ $size: endpoint }, 0] }, { $ne: [endpoint, null] }],+/** Graphs read per cleanup pass; bounds application memory the way the server-side update did. */+export const EDGE_CLEANUP_BATCH = 200;+/** Consecutive passes that may clean nothing before the loop gives up. */+const EDGE_CLEANUP_STALLED_PASSES = 5;+/** Sweeps from the top before the loop gives up on references that keep being added. */+export const EDGE_CLEANUP_MAX_SWEEPS = 5;++type AgentEdge = NonNullable<IAgent['edges']>[number];+type EdgeEndpoint = AgentEdge['from'];++/** An endpoint with `removed` taken out: a list loses those ids; a single id that is one becomes null. */+function pruneEndpoint(+ endpoint: EdgeEndpoint | null | undefined,+ removed: Set<string>,+): EdgeEndpoint | null {+ if (Array.isArray(endpoint)) {+ return endpoint.filter((id) => !removed.has(id));+ }+ return typeof endpoint === 'string' && removed.has(endpoint) ? null : (endpoint ?? null);+}++function hasEndpoint(endpoint: EdgeEndpoint | null): endpoint is EdgeEndpoint {+ return Array.isArray(endpoint) ? endpoint.length > 0 : endpoint != null;+}++/** The edges that survive removing `removed`, with those ids pruned from their endpoints. */+export function pruneEdges(edges: IAgent['edges'], removed: Set<string>): AgentEdge[] {+ return (edges ?? []).flatMap((edge) => {+ const from = pruneEndpoint(edge.from, removed);+ const to = pruneEndpoint(edge.to, removed);+ return hasEndpoint(from) && hasEndpoint(to) ? [{ ...edge, from, to }] : [];});+}- return [- {- $set: {- edges: {- $filter: {- input: {- $map: {- input: { $ifNull: ['$edges', []] },- as: 'edge',- in: {- $let: {- vars: {- cleanedFrom: cleanEndpoint('$$edge.from'),- cleanedTo: cleanEndpoint('$$edge.to'),- },- in: {- $cond: [- {- $and: [hasEndpoint('$$cleanedFrom'), hasEndpoint('$$cleanedTo')],- },- {- $mergeObjects: [- '$$edge',- {- from: '$$cleanedFrom',- to: '$$cleanedTo',- },- ],- },- null,- ],- },- },- },- },- },- as: 'edge',- cond: { $ne: ['$$edge', null] },- },- },- },- },- ];+interface GraphEdges {+ _id: Types.ObjectId;+ edges?: IAgent['edges'];}-/** Removes deleted agent references from active graphs in the requested tenant. */+/**+ * Removes deleted agent references from active graphs in the requested tenant.+ * Graphs are read a page at a time behind an `_id` cursor, and each page's+ * pruned edges are written back behind a compare-and-set on the edges that were+ * read, so a concurrent edit is never overwritten. A graph whose edges changed+ * underneath fails its compare-and-set and is left behind the cursor, so once+ * the cursor is exhausted one more sweep from the top picks up every miss (and+ * any reference added meanwhile); the cleanup ends when a sweep from the top+ * finds nothing, and gives up after a bounded number of sweeps if references+ * keep being added. This is the plain-operator form of what was an+ * aggregation-pipeline update, which Amazon DocumentDB rejects.+ */async function removeAgentIdsFromEdges(Agent: Model<IAgent>,agentIds: string[],@@ -118,14 +105,51 @@ async function removeAgentIdsFromEdges(if (agentIds.length === 0) {return;}-- await Agent.updateMany(- {- ...(tenantId !== undefined ? { tenantId } : {}),- $or: [{ 'edges.from': { $in: agentIds } }, { 'edges.to': { $in: agentIds } }],- },- createEdgeCleanupPipeline(agentIds),- );+ const filter: FilterQuery<IAgent> = {+ ...(tenantId !== undefined ? { tenantId } : {}),+ $or: [{ 'edges.from': { $in: agentIds } }, { 'edges.to': { $in: agentIds } }],+ };+ const removed = new Set(agentIds);+ let stalledPasses = 0;+ let sweeps = 0;+ let after: Types.ObjectId | undefined;+ for (;;) {+ const graphs = await Agent.find(after == null ? filter : { ...filter, _id: { $gt: after } })+ .sort({ _id: 1 })+ .limit(EDGE_CLEANUP_BATCH)+ .select('_id edges')+ .lean<GraphEdges[]>();+ if (graphs.length === 0) {+ if (after == null) {+ return;+ }+ sweeps += 1;+ if (sweeps >= EDGE_CLEANUP_MAX_SWEEPS) {+ throw new Error(+ `[removeAgentIdsFromEdges] references kept being added during cleanup (${EDGE_CLEANUP_MAX_SWEEPS} sweeps)`,+ );+ }+ after = undefined;+ continue;+ }+ const result = await tenantSafeBulkWrite(+ Agent,+ graphs.map((graph) => ({+ updateOne: {+ filter: { _id: graph._id, edges: graph.edges },+ update: { $set: { edges: pruneEdges(graph.edges, removed) } },+ },+ })),+ { ordered: false },+ );+ stalledPasses = result.matchedCount === 0 ? stalledPasses + 1 : 0;+ if (stalledPasses >= EDGE_CLEANUP_STALLED_PASSES) {+ throw new Error(+ `[removeAgentIdsFromEdges] graph edges kept changing during cleanup (${EDGE_CLEANUP_STALLED_PASSES} passes without progress)`,+ );+ }+ after = graphs[graphs.length - 1]._id;+ }}export interface AgentDeps {diff --git a/packages/data-schemas/src/methods/auditLog.spec.ts b/packages/data-schemas/src/methods/auditLog.spec.tsindex aefdf34e81..9ae3262696 100644--- a/packages/data-schemas/src/methods/auditLog.spec.ts+++ b/packages/data-schemas/src/methods/auditLog.spec.ts@@ -506,6 +506,32 @@ describe('auditLog methods', () => {});describe('append index safety', () => {+ it('fails open within the index-build deadline while a peer holds the collection', async () => {+ const prototype = mongoose.mongo.Collection.prototype;+ const createIndex = prototype.createIndex;+ const collectionName = AuditLog.collection.name;+ prototype.createIndex = async function (this: mongoose.mongo.Collection, spec, options) {+ if (this.collectionName !== collectionName) {+ return createIndex.call(this, spec, options);+ }+ throw new mongoose.mongo.MongoServerError({+ ok: 0,+ code: 40333,+ errmsg:+ 'Existing index build in progress on the same collection. Collection is limited to a single index build at a time.',+ });+ };+ try {+ const freshMethods = createAuditLogMethods(mongoose, {+ indexBuild: { peerBuildPollMs: 1, peerBuildDeadlineMs: 20 },+ });++ await expect(freshMethods.recordAuditEntry(baseInput())).resolves.toBeNull();+ } finally {+ prototype.createIndex = createIndex;+ }+ });+it('builds the unique seq index before appending (independent of autoIndex)', async () => {const spy = jest.spyOn(AuditLog, 'createIndexes');// a fresh methods instance has not yet memoized the index builddiff --git a/packages/data-schemas/src/methods/auditLog.ts b/packages/data-schemas/src/methods/auditLog.tsindex 0ab6a7dd51..d645b9c574 100644--- a/packages/data-schemas/src/methods/auditLog.ts+++ b/packages/data-schemas/src/methods/auditLog.ts@@ -14,7 +14,9 @@ import type {RecordAuditEntryOptions,VerifyAuditChainOptions,} from '~/types';+import type { IndexBuildOptions } from '~/utils/retry';import { GENESIS_HASH, PLATFORM_CHAIN_KEY } from '~/schema/auditLog';+import { createIndexesWithRetry } from '~/utils/retry';import { AUDIT_ACTION_CATEGORY } from '~/types/admin';import logger from '~/config/winston';@@ -290,7 +292,18 @@ function buildFilter(chainKey: string, filters: AuditLogFilters): FilterQuery<IAreturn query;}-export function createAuditLogMethods(mongoose: typeof import('mongoose')): AuditLogMethods {+/** Longest wait on the request path for a peer's index build before an append fails open. */+const AUDIT_INDEX_BUILD_DEADLINE_MS = 10_000;++export interface AuditLogMethodOptions {+ /** Overrides for the append index build; tests shorten its polling and deadline. */+ indexBuild?: IndexBuildOptions;+}++export function createAuditLogMethods(+ mongoose: typeof import('mongoose'),+ options: AuditLogMethodOptions = {},+): AuditLogMethods {function model(): Model<IAuditLog> {return mongoose.models.AuditLog as Model<IAuditLog>;}@@ -301,18 +314,22 @@ export function createAuditLogMethods(mongoose: typeof import('mongoose')): Audi* during the startup window before a background build finishes, two writers* could insert the same `seq` with no duplicate-key error and silently fork* the chain. Build the indexes once before the first append so serialization- * never depends on a background build. Memoized; reset on failure so a later- * write retries.+ * never depends on a background build — letting Mongoose's own background+ * build settle first, so the two never overlap on a single-build engine. The+ * wait for a peer's build is bounded, because this runs on the request path:+ * past the deadline the append fails open below instead of holding the+ * request. Memoized; reset on failure so a later write retries.*/let indexPromise: Promise<unknown> | null = null;function ensureIndexes(): Promise<unknown> {if (!indexPromise) {- indexPromise = model()- .createIndexes()- .catch((err) => {- indexPromise = null;- throw err;- });+ indexPromise = createIndexesWithRetry(model(), {+ peerBuildDeadlineMs: AUDIT_INDEX_BUILD_DEADLINE_MS,+ ...options.indexBuild,+ }).catch((err) => {+ indexPromise = null;+ throw err;+ });}return indexPromise;}diff --git a/packages/data-schemas/src/methods/documentdb.spec.ts b/packages/data-schemas/src/methods/documentdb.spec.tsindex 4cc139514d..e898016af0 100644--- a/packages/data-schemas/src/methods/documentdb.spec.ts+++ b/packages/data-schemas/src/methods/documentdb.spec.ts@@ -15,9 +15,16 @@ import ts from 'typescript';** The scan walks the TypeScript AST rather than source text, so it also* catches a pipeline bound to a variable first, cast with `as`, or nested in a- * `bulkWrite` operation's `update` property. It covers every backend workspace+ * `bulkWrite` operation's `update` property. Besides constructs the engine+ * rejects outright, it holds every index build to the retrying helpers in+ * `utils/retry.ts`, because the engine admits one build per collection at a+ * time and a concurrent second build fails silently in any caller that only+ * logs. It covers every backend workspace* that talks to MongoDB — this package, `packages/api`, and `api` — because the* regression class is repo-wide and new backend code lands in `packages/api`.+ * Dataflow is followed within a file only — a pipeline imported from another+ * module is out of reach — and the method sweep and the live cluster run are+ * the completeness backstops, not this guard.* If a construct here becomes genuinely necessary, the fix is a compatible* rewrite, not an exception list: `misc/documentdb/audit.documentdb.spec.ts`* re-adjudicates any of this against a real cluster.@@ -40,8 +47,123 @@ const UPDATE_METHODS = new Set(['replaceOne',]);-/** `$$REMOVE`: `Feature not supported: $$REMOVE`. `$facet`: `Aggregation stage not supported`. */-const FORBIDDEN_TOKENS = ['$$REMOVE', '$$CURRENT', '$facet', '$graphLookup', '$unionWith'];+/**+ * Every operator AWS documents as unsupported in the 5.0 column of "Supported+ * MongoDB APIs, operations, and data types", grouped as that table groups them.+ * The first two carry the exact server errors observed live — `$$REMOVE`:+ * `Feature not supported: $$REMOVE`; `$facet`: `Aggregation stage not+ * supported`. `$pow`, `$rand`, `$dateFromParts` and `$dateToParts` arrive in+ * 5.0.1, after the 5.0.0 engine these verdicts were established on. `$set`,+ * `$unset` and `$count` are legal in one position and rejected in another, so+ * they have their own detectors below.+ */+const FORBIDDEN_TOKENS = [+ '$$REMOVE',+ '$$CURRENT',+ // stages+ '$facet',+ '$graphLookup',+ '$unionWith',+ '$setWindowFields',+ '$bucket',+ '$bucketAuto',+ '$merge',+ '$replaceWith',+ '$sortByCount',+ '$vectorSearch',+ '$listSearchIndexes',+ '$planCacheStats',+ '$listSessions',+ '$listLocalSessions',+ // accumulators+ '$accumulator',+ '$rank',+ '$denseRank',+ '$documentNumber',+ '$shift',+ '$derivative',+ '$integral',+ '$expMovingAvg',+ '$covariancePop',+ '$covarianceSamp',+ '$stdDevPop',+ '$stdDevSamp',+ '$top',+ '$topN',+ '$bottom',+ '$bottomN',+ '$firstN',+ '$lastN',+ '$maxN',+ '$minN',+ '$median',+ '$percentile',+ // expressions+ '$round',+ '$trunc',+ '$pow',+ '$rand',+ '$sortArray',+ '$binarySize',+ '$bsonSize',+ '$dateTrunc',+ '$dateFromParts',+ '$dateToParts',+ '$isNumber',+ '$toUUID',+ '$getField',+ '$sampleRate',+ '$sigmoid',+ '$bitAnd',+ '$bitNot',+ '$bitOr',+ '$bitXor',+ '$tsIncrement',+ '$tsSecond',+ '$acos',+ '$acosh',+ '$asin',+ '$asinh',+ '$atan',+ '$atan2',+ '$atanh',+ '$cos',+ '$cosh',+ '$sin',+ '$sinh',+ '$tan',+ '$tanh',+ '$degreesToRadians',+ '$radiansToDegrees',+ // query and cursor options+ '$where',+ 'allowDiskUse',+ 'noCursorTimeout',+];++/** `$set` and `$unset` are update operators everywhere in this codebase and are+ * supported as such; as pipeline STAGES (the `$addFields` / `$project` aliases)+ * DocumentDB 5.0 rejects them. A stage is an object that is an element of an+ * array, which no update-operator position is: an update document is a call+ * argument or an `update` property, and a `bulkWrite` element keys on its+ * operation name. */+const ALIAS_STAGES = new Set(['$set', '$unset']);++/** Index builds. The engine admits one build per collection at a time and+ * rejects a second with code 40333 (`utils/retry.ts`), so every build must go+ * through `createIndexesWithRetry` or `buildIndexWithRetry`, which poll for the+ * slot. A raw call is legal only inside the helper module itself or as the+ * argument of `buildIndexWithRetry`. */+const INDEX_BUILD_METHODS = new Set([+ 'createIndex',+ 'createIndexes',+ 'syncIndexes',+ 'ensureIndexes',+]);+const INDEX_BUILD_HELPER = 'packages/data-schemas/src/utils/retry.ts';+const INDEX_BUILD_WRAPPER = 'buildIndexWithRetry';+/** Modules that name forbidden operators in order to reject them. */+const OPERATOR_GUARDS = new Set(['packages/data-schemas/src/tenant/probe.ts']);function collectSourceFiles(directory: string, found: string[] = []): string[] {for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {@@ -78,25 +200,141 @@ function unwrapExpression(expression: ts.Expression): ts.Expression {return current;}-/** Names of variables initialized with array literals, so an indirect- * `const update = [...]; Model.updateOne(filter, update)` is still caught.- * Scope-naive by design: a false positive here names a variable that holds an- * array and is passed as an update, which deserves a look regardless. */-function collectArrayVariableNames(sourceFile: ts.SourceFile): Set<string> {- const names = new Set<string>();+function isArrayType(type: ts.TypeNode | undefined): boolean {+ if (type == null) {+ return false;+ }+ if (ts.isArrayTypeNode(type)) {+ return true;+ }+ return (+ ts.isTypeReferenceNode(type) && ts.isIdentifier(type.typeName) && type.typeName.text === 'Array'+ );+}++function isArrayExpression(expression: ts.Expression, arrayNames: Set<string>): boolean {+ const unwrapped = unwrapExpression(expression);+ return (+ ts.isArrayLiteralExpression(unwrapped) ||+ (ts.isIdentifier(unwrapped) && arrayNames.has(unwrapped.text))+ );+}++/** Whether a function is declared to return an array, or returns one from its+ * own body — an array literal, or a variable bound to one (nested functions+ * are not descended into). */+function returnsArray(fn: ts.FunctionLikeDeclaration, arrayNames: Set<string>): boolean {+ if (isArrayType(fn.type)) {+ return true;+ }+ if (fn.body == null) {+ return false;+ }+ if (!ts.isBlock(fn.body)) {+ return isArrayExpression(fn.body, arrayNames);+ }+ let found = false;const visit = (node: ts.Node): void => {+ if (found || ts.isFunctionLike(node)) {+ return;+ }+ if (+ ts.isReturnStatement(node) &&+ node.expression != null &&+ isArrayExpression(node.expression, arrayNames)+ ) {+ found = true;+ return;+ }+ ts.forEachChild(node, visit);+ };+ ts.forEachChild(fn.body, visit);+ return found;+}++function isArrayReturningFunction(expression: ts.Expression, arrayNames: Set<string>): boolean {+ return (+ (ts.isArrowFunction(expression) || ts.isFunctionExpression(expression)) &&+ returnsArray(expression, arrayNames)+ );+}++/** Names of variables initialized with array literals, then of functions and+ * methods that return one — an array literal or one of those variables — so an+ * indirect `const update = [...]; Model.updateOne(filter, update)`, a+ * `Model.updateMany(filter, buildPipeline(ids))` or a `builder.pipeline()` is+ * still caught. Scope-naive by design: a false positive here names something+ * that holds an array and is passed as an update, which deserves a look+ * regardless. */+function collectArrayValuedNames(sourceFile: ts.SourceFile): Set<string> {+ const names = new Set<string>();+ const visitVariables = (node: ts.Node): void => {+ if (+ ts.isVariableDeclaration(node) &&+ ts.isIdentifier(node.name) &&+ (isArrayType(node.type) ||+ (node.initializer != null &&+ ts.isArrayLiteralExpression(unwrapExpression(node.initializer))))+ ) {+ names.add(node.name.text);+ }+ ts.forEachChild(node, visitVariables);+ };+ visitVariables(sourceFile);+ const visitFunctions = (node: ts.Node): void => {if (ts.isVariableDeclaration(node) &&ts.isIdentifier(node.name) &&node.initializer != null &&- ts.isArrayLiteralExpression(unwrapExpression(node.initializer))+ isArrayReturningFunction(unwrapExpression(node.initializer), names)) {names.add(node.name.text);}+ if (+ (ts.isFunctionDeclaration(node) || ts.isMethodDeclaration(node)) &&+ node.name != null &&+ ts.isIdentifier(node.name) &&+ returnsArray(node, names)+ ) {+ names.add(node.name.text);+ }+ if (+ ts.isPropertyAssignment(node) &&+ ts.isIdentifier(node.name) &&+ isArrayReturningFunction(unwrapExpression(node.initializer), names)+ ) {+ names.add(node.name.text);+ }+ ts.forEachChild(node, visitFunctions);+ };+ visitFunctions(sourceFile);+ return names;+}++/** Object literals bound to variable names, so a stage held in a variable and+ * pushed or listed later is still inspected. Scope-naive like the array names. */+function collectObjectValuedNames(+ sourceFile: ts.SourceFile,+): Map<string, ts.ObjectLiteralExpression> {+ const objects = new Map<string, ts.ObjectLiteralExpression>();+ const visit = (node: ts.Node): void => {+ if (ts.isVariableDeclaration(node) && ts.isIdentifier(node.name) && node.initializer != null) {+ const initializer = unwrapExpression(node.initializer);+ if (ts.isObjectLiteralExpression(initializer)) {+ objects.set(node.name.text, initializer);+ }+ }ts.forEachChild(node, visit);};visit(sourceFile);- return names;+ return objects;+}++function calleeName(call: ts.CallExpression): string | undefined {+ if (ts.isIdentifier(call.expression)) {+ return call.expression.text;+ }+ return ts.isPropertyAccessExpression(call.expression) ? call.expression.name.text : undefined;}function isArrayValued(expression: ts.Expression, arrayNames: Set<string>): boolean {@@ -107,6 +345,10 @@ function isArrayValued(expression: ts.Expression, arrayNames: Set<string>): boolif (ts.isIdentifier(unwrapped)) {return arrayNames.has(unwrapped.text);}+ if (ts.isCallExpression(unwrapped)) {+ const name = calleeName(unwrapped);+ return name != null && arrayNames.has(name);+ }if (ts.isConditionalExpression(unwrapped)) {return (isArrayValued(unwrapped.whenTrue, arrayNames) ||@@ -125,7 +367,7 @@ function offenseAt(sourceFile: ts.SourceFile, node: ts.Node, label: string): str* whether passed directly to an update method or carried inside a `bulkWrite`* operation's `update` property. */function findPipelineUpdates(sourceFile: ts.SourceFile): string[] {- const arrayNames = collectArrayVariableNames(sourceFile);+ const arrayNames = collectArrayValuedNames(sourceFile);const offenses: string[] = [];const visit = (node: ts.Node): void => {if (@@ -159,11 +401,19 @@ function findPipelineUpdates(sourceFile: ts.SourceFile): string[] {}/** Reports forbidden operator tokens in string literals and property names,- * ignoring prose — the rewrites explain themselves by naming the construct. */+ * ignoring prose — the rewrites explain themselves by naming the construct —+ * and type members, which never reach the engine (Mongoose documents declare+ * a `$where` field). */function findForbiddenTokens(sourceFile: ts.SourceFile): string[] {+ if (OPERATOR_GUARDS.has(sourceFile.fileName)) {+ return [];+ }const offenses: string[] = [];const visit = (node: ts.Node): void => {- if (ts.isStringLiteralLike(node) || ts.isIdentifier(node)) {+ if (+ ts.isStringLiteralLike(node) ||+ (ts.isIdentifier(node) && !ts.isPropertySignature(node.parent))+ ) {for (const token of FORBIDDEN_TOKENS) {if (node.text === token || node.text.startsWith(`${token}.`)) {offenses.push(offenseAt(sourceFile, node, token));@@ -212,10 +462,139 @@ function findMixedSelectStrings(sourceFile: ts.SourceFile): string[] {return offenses;}+function propertyName(property: ts.ObjectLiteralElementLike): string | undefined {+ const name = property.name;+ if (name == null || !(ts.isIdentifier(name) || ts.isStringLiteral(name))) {+ return undefined;+ }+ return name.text;+}++/** Array methods whose object arguments become elements of the receiver. */+const ARRAY_APPENDERS = new Set(['push', 'unshift', 'concat']);++function aliasStageOffenses(+ sourceFile: ts.SourceFile,+ element: ts.Expression,+ objects: Map<string, ts.ObjectLiteralExpression>,+): string[] {+ const unwrapped = unwrapExpression(element);+ if (ts.isConditionalExpression(unwrapped)) {+ return [+ ...aliasStageOffenses(sourceFile, unwrapped.whenTrue, objects),+ ...aliasStageOffenses(sourceFile, unwrapped.whenFalse, objects),+ ];+ }+ const stage = ts.isIdentifier(unwrapped) ? objects.get(unwrapped.text) : unwrapped;+ if (stage == null || !ts.isObjectLiteralExpression(stage)) {+ return [];+ }+ return stage.properties+ .filter((property) => ALIAS_STAGES.has(propertyName(property) ?? ''))+ .map((property) => offenseAt(sourceFile, property, `${propertyName(property)} stage`));+}++/** The expressions that become elements of an array: the elements of a literal,+ * and the arguments of `push` / `unshift` / `concat`, which is how a pipeline+ * is assembled dynamically. */+function arrayElements(node: ts.Node): readonly ts.Expression[] {+ if (ts.isArrayLiteralExpression(node)) {+ return node.elements;+ }+ if (+ ts.isCallExpression(node) &&+ ts.isPropertyAccessExpression(node.expression) &&+ ARRAY_APPENDERS.has(node.expression.name.text)+ ) {+ return node.arguments;+ }+ return [];+}++/** Reports `$set` / `$unset` objects in element position — inside an array+ * literal or appended to one, on either side of a conditional — wherever the+ * pipeline is assembled. */+function findAliasStages(sourceFile: ts.SourceFile): string[] {+ const objects = collectObjectValuedNames(sourceFile);+ const offenses: string[] = [];+ const visit = (node: ts.Node): void => {+ offenses.push(+ ...arrayElements(node).flatMap((element) => aliasStageOffenses(sourceFile, element, objects)),+ );+ ts.forEachChild(node, visit);+ };+ visit(sourceFile);+ return offenses;+}++/** `$count` is a supported STAGE (`{ $count: 'total' }`) but an unsupported+ * ACCUMULATOR (`{ n: { $count: {} } }`) on 5.0; the two differ by value shape. */+function findAccumulatorCounts(sourceFile: ts.SourceFile): string[] {+ const offenses: string[] = [];+ const visit = (node: ts.Node): void => {+ if (+ ts.isPropertyAssignment(node) &&+ propertyName(node) === '$count' &&+ ts.isObjectLiteralExpression(unwrapExpression(node.initializer))+ ) {+ offenses.push(offenseAt(sourceFile, node, '$count accumulator'));+ }+ ts.forEachChild(node, visit);+ };+ visit(sourceFile);+ return offenses;+}++function isWrappedIndexBuild(node: ts.Node): boolean {+ for (let current = node.parent; current != null; current = current.parent) {+ if (+ ts.isCallExpression(current) &&+ ts.isIdentifier(current.expression) &&+ current.expression.text === INDEX_BUILD_WRAPPER+ ) {+ return true;+ }+ }+ return false;+}++/** Meilisearch's client shares the `createIndex` name; its options carry a+ * `primaryKey`, which no MongoDB index option does. */+function isMeilisearchIndexCall(call: ts.CallExpression): boolean {+ const options = call.arguments[1] == null ? undefined : unwrapExpression(call.arguments[1]);+ return (+ options != null &&+ ts.isObjectLiteralExpression(options) &&+ options.properties.some((property) => propertyName(property) === 'primaryKey')+ );+}++/** Reports every index build that bypasses the retrying helpers. */+function findRawIndexBuilds(sourceFile: ts.SourceFile): string[] {+ if (sourceFile.fileName === INDEX_BUILD_HELPER) {+ return [];+ }+ const offenses: string[] = [];+ const visit = (node: ts.Node): void => {+ if (+ ts.isCallExpression(node) &&+ ts.isPropertyAccessExpression(node.expression) &&+ INDEX_BUILD_METHODS.has(node.expression.name.text) &&+ !isWrappedIndexBuild(node) &&+ !isMeilisearchIndexCall(node)+ ) {+ offenses.push(offenseAt(sourceFile, node, `raw ${node.expression.name.text}`));+ }+ ts.forEachChild(node, visit);+ };+ visit(sourceFile);+ return offenses;+}+describe('Amazon DocumentDB compatibility', () => {- /** Each file is read and parsed once; both detectors walk the same tree. */+ /** Each file is read and parsed once; every detector walks the same tree. */const parsedSources = SCAN_ROOTS.flatMap((root) => collectSourceFiles(root)).map((file) =>- parse(path.relative(REPO_ROOT, file), fs.readFileSync(file, 'utf8')),+ parse(path.relative(REPO_ROOT, file).split(path.sep).join('/'), fs.readFileSync(file, 'utf8')),);it('scans the backend workspaces', () => {@@ -234,6 +613,18 @@ describe('Amazon DocumentDB compatibility', () => {expect(parsedSources.flatMap(findMixedSelectStrings)).toEqual([]);});+ it('uses no $set or $unset pipeline stages', () => {+ expect(parsedSources.flatMap(findAliasStages)).toEqual([]);+ });++ it('uses no $count accumulators', () => {+ expect(parsedSources.flatMap(findAccumulatorCounts)).toEqual([]);+ });++ it('builds every index through the retrying helpers', () => {+ expect(parsedSources.flatMap(findRawIndexBuilds)).toEqual([]);+ });+/** A guard that cannot fail protects nothing, so every shape the detectors* exist to catch is proven against fixtures here rather than by trusting a* one-time manual injection. */@@ -257,6 +648,34 @@ describe('Amazon DocumentDB compatibility', () => {'bulkWrite indirect payload',`const update = [{ $set: { a: 1 } }];\nawait Model.bulkWrite([{ updateMany: { filter, update } }]);`,],+ [+ 'pipeline returned by a declared function',+ `function pipeline(ids: string[]): PipelineStage[] {\n return [{ $set: { ids } }];\n}\nModel.updateMany(filter, pipeline(ids));`,+ ],+ [+ 'pipeline returned by an arrow with an expression body',+ `const pipeline = (ids: string[]) => [{ $set: { ids } }];\nModel.updateOne(filter, pipeline(ids));`,+ ],+ [+ 'pipeline returned by an untyped function body',+ `function pipeline() {\n const stage = { $set: { a: 1 } };\n return [stage] as PipelineStage[];\n}\nModel.findOneAndUpdate(filter, pipeline());`,+ ],+ [+ 'pipeline returned by a class method',+ `class Builder {\n pipeline(): PipelineStage[] {\n return [{ $addFields: { a: 1 } }];\n }\n}\nModel.updateMany(filter, new Builder().pipeline());`,+ ],+ [+ 'pipeline returned by an object method',+ `const builder = { pipeline: () => [{ $addFields: { a: 1 } }] };\nModel.updateOne(filter, builder.pipeline());`,+ ],+ [+ 'pipeline returned through a local variable',+ `function pipeline() {\n const stages = [{ $addFields: { a: 1 } }];\n return stages;\n}\nModel.updateMany(filter, pipeline());`,+ ],+ [+ 'annotated variable with a builder initializer',+ `const update: PipelineStage[] = importedBuilder();\nModel.updateMany(filter, update);`,+ ],])('flags a pipeline update: %s', (_shape, source) => {expect(findPipelineUpdates(parse('fixture.ts', source))).not.toEqual([]);});@@ -268,6 +687,10 @@ describe('Amazon DocumentDB compatibility', () => {`await Model.bulkWrite([{ updateOne: { filter, update: { $set: { a: 1 } } } }]);`,],['unrelated array variable', `const stages = [{ $match: {} }];\nModel.aggregate(stages);`],+ [+ 'update document returned by a function',+ `function update() {\n return { $set: { a: 1 } };\n}\nModel.updateOne(filter, update());`,+ ],])('accepts a supported shape: %s', (_shape, source) => {expect(findPipelineUpdates(parse('fixture.ts', source))).toEqual([]);});@@ -289,16 +712,96 @@ describe('Amazon DocumentDB compatibility', () => {expect(findMixedSelectStrings(parse('fixture.ts', source))).toEqual([]);});- it('flags forbidden operators in code but not in prose', () => {+ it('flags forbidden operators in code but not in prose or type members', () => {expect(findForbiddenTokens(parse('fixture.ts', `const projection = { x: '$$REMOVE' };`)),).not.toEqual([]);expect(findForbiddenTokens(parse('fixture.ts', `pipeline.push({ $facet: { rows: [] } });`)),).not.toEqual([]);+ expect(+ findForbiddenTokens(parse('fixture.ts', `const stage = { $setWindowFields: {} };`)),+ ).not.toEqual([]);expect(findForbiddenTokens(parse('fixture.ts', `/** $$REMOVE and $facet are unsupported. */`)),).toEqual([]);+ expect(+ findForbiddenTokens(+ parse('fixture.ts', `interface Doc { $where: Record<string, unknown> }`),+ ),+ ).toEqual([]);+ });++ it.each([+ ['stage after a match', `Model.aggregate([{ $match: {} }, { $set: { a: 1 } }]);`],+ ['unset stage held in a variable', `const stages = [{ $unset: 'a' }];`],+ ['stage appended to a spread scope', `Model.aggregate([...scope, { $set: { a: 1 } }]);`],+ [+ 'stage pushed onto a pipeline',+ `const stages = [];\nstages.push({ $match: {} }, { $set: { a: 1 } });\nModel.aggregate(stages);`,+ ],+ ['stage prepended with unshift', `stages.unshift({ $unset: 'a' });`],+ ['stage concatenated onto a base', `const stages = base.concat({ $set: { a: 1 } });`],+ [+ 'stage chosen by a conditional element',+ `Model.aggregate([...scope, flag ? { $set: { a: 1 } } : { $addFields: { a: 1 } }]);`,+ ],+ [+ 'stage held in a variable and pushed',+ `const stage = { $set: { a: 1 } };\nstages.push(stage);`,+ ],+ [+ 'stage held in a variable and listed',+ `const stage = { $unset: 'a' };\nconst stages = [stage];`,+ ],+ ])('flags an alias stage: %s', (_shape, source) => {+ expect(findAliasStages(parse('fixture.ts', source))).not.toEqual([]);+ });++ it.each([+ ['classic update operator', `Model.updateOne(filter, { $set: { a: 1 } });`],+ [+ 'bulkWrite operation payload',+ `await Model.bulkWrite([{ updateOne: { filter, update: { $set: { a: 1 } } } }]);`,+ ],+ ['addFields stage', `Model.aggregate([{ $addFields: { a: 1 } }]);`],+ [+ 'bulk operation pushed onto an operations list',+ `const operations = [];\noperations.push({ updateOne: { filter, update: { $set: { a: 1 } } } });`,+ ],+ ])('accepts a supported $set position: %s', (_shape, source) => {+ expect(findAliasStages(parse('fixture.ts', source))).toEqual([]);+ });++ it('flags a $count accumulator but not the $count stage', () => {+ expect(+ findAccumulatorCounts(+ parse('fixture.ts', `Model.aggregate([{ $group: { _id: null, n: { $count: {} } } }]);`),+ ),+ ).not.toEqual([]);+ expect(+ findAccumulatorCounts(parse('fixture.ts', `Model.aggregate([{ $count: 'total' }]);`)),+ ).toEqual([]);+ });++ it.each([+ ['raw createIndex', `await collection.createIndex({ a: 1 });`],+ ['raw createIndex with options', `await collection.createIndex({ a: 1 }, { unique: true });`],+ ['raw createIndexes', `await Model.createIndexes();`],+ ['raw syncIndexes', `await Model.syncIndexes();`],+ ])('flags an unguarded index build: %s', (_shape, source) => {+ expect(findRawIndexBuilds(parse('fixture.ts', source))).not.toEqual([]);+ });++ it.each([+ [+ 'a build wrapped by the helper',+ `await buildIndexWithRetry(() => collection.createIndex({ a: 1 }), 'a_1');`,+ ],+ ['the model helper', `await createIndexesWithRetry(Model);`],+ ['the Meilisearch client', `await client.createIndex('messages', { primaryKey: 'id' });`],+ ])('accepts a guarded index build: %s', (_shape, source) => {+ expect(findRawIndexBuilds(parse('fixture.ts', source))).toEqual([]);});});});diff --git a/packages/data-schemas/src/migrations/mcpAuthorityIndexes.ts b/packages/data-schemas/src/migrations/mcpAuthorityIndexes.tsindex 4062dc691c..74b08a52bf 100644--- a/packages/data-schemas/src/migrations/mcpAuthorityIndexes.ts+++ b/packages/data-schemas/src/migrations/mcpAuthorityIndexes.ts@@ -1,5 +1,6 @@import type { IndexSpecification } from 'mongodb';import type { Connection } from 'mongoose';+import { buildIndexWithRetry } from '~/utils/retry';interface AuthorityIndexDefinition {collection: string;@@ -36,9 +37,11 @@ export async function createMCPAuthorityLookupIndexes(): Promise<readonly string[]> {const created: string[] = [];for (const definition of AUTHORITY_INDEXES) {- const name = await connection- .db!.collection(definition.collection)- .createIndex(definition.keys, { name: definition.name });+ const collection = connection.db!.collection(definition.collection);+ const name = await buildIndexWithRetry(+ () => collection.createIndex(definition.keys, { name: definition.name }),+ `createIndex(${definition.collection}.${definition.name})`,+ );created.push(name);}return created;diff --git a/packages/data-schemas/src/migrations/mcpServerNames.ts b/packages/data-schemas/src/migrations/mcpServerNames.tsindex 8be0f51b45..c8f47a1120 100644--- a/packages/data-schemas/src/migrations/mcpServerNames.ts+++ b/packages/data-schemas/src/migrations/mcpServerNames.ts@@ -1,5 +1,6 @@import { normalizeServerName } from 'librechat-data-provider';import type { Connection, Types } from 'mongoose';+import { buildIndexWithRetry } from '~/utils/retry';interface MCPServerNameRow {_id: Types.ObjectId;@@ -96,13 +97,17 @@ export async function backfillMCPServerNormalizedNames(// eslint-disable-next-line no-restricted-syntax -- offline all-tenant migration intentionally bypasses request tenant scopingawait collection.bulkWrite(updates, { ordered: true });}- await collection.createIndex(- { normalizedServerName: 1, tenantId: 1 },- {- name: 'normalizedServerName_1_tenantId_1',- unique: true,- partialFilterExpression: { normalizedServerName: { $exists: true } },- },+ await buildIndexWithRetry(+ () =>+ collection.createIndex(+ { normalizedServerName: 1, tenantId: 1 },+ {+ name: 'normalizedServerName_1_tenantId_1',+ unique: true,+ partialFilterExpression: { normalizedServerName: { $exists: true } },+ },+ ),+ 'createIndex(normalizedServerName_1_tenantId_1)',);return { scanned, updated };}diff --git a/packages/data-schemas/src/utils/index.ts b/packages/data-schemas/src/utils/index.tsindex 4adfa3488b..fecdfcc7ed 100644--- a/packages/data-schemas/src/utils/index.ts+++ b/packages/data-schemas/src/utils/index.ts@@ -8,3 +8,5 @@ export * from './objectId';export * from './yaml';export * from './stripUIResourceMarkers';export * from './fading';+export { buildIndexWithRetry, createIndexesWithRetry, isIndexBuildInProgress } from './retry';+export type { IndexBuildOptions } from './retry';diff --git a/packages/data-schemas/src/utils/retry.spec.ts b/packages/data-schemas/src/utils/retry.spec.tsindex a7dbb257c6..6bc33ddbb5 100644--- a/packages/data-schemas/src/utils/retry.spec.ts+++ b/packages/data-schemas/src/utils/retry.spec.ts@@ -2,8 +2,8 @@ import mongoose from 'mongoose';import { MongoMemoryServer } from 'mongodb-memory-server';import type { Model } from 'mongoose';import type { IAgentTriggerDeliveryDocument } from '~/types/triggerDelivery';+import { buildIndexWithRetry, createIndexesWithRetry, retryWithBackoff } from './retry';import triggerDeliverySchema from '~/schema/triggerDelivery';-import { createIndexesWithRetry } from './retry';const DB_SETUP_TIMEOUT_MS = 60_000;/** Amazon DocumentDB: "Existing index build in progress on the same collection." */@@ -154,3 +154,114 @@ describe('createIndexesWithRetry on a single-index-build engine', () => {expect(attempts).toBe(triggerDeliverySchema.indexes().length);});});++describe('buildIndexWithRetry on a raw driver collection', () => {+ function rawCollection() {+ modelCounter += 1;+ return mongoose.connection.db!.collection(`retry_spec_raw_${modelCounter}`);+ }++ test('keeps polling while another replica holds the collection, then builds', async () => {+ const collection = rawCollection();+ const createIndex = collection.createIndex.bind(collection);+ let remainingRejections = 3;+ let attempts = 0;+ collection.createIndex = async (fields, options) => {+ attempts += 1;+ if (remainingRejections > 0) {+ remainingRejections -= 1;+ throw indexBuildInProgressError();+ }+ return createIndex(fields, options);+ };++ await expect(+ buildIndexWithRetry(() => collection.createIndex({ key: 1 }, { name: 'key_1' }), 'key_1', {+ maxAttempts: 1,+ peerBuildPollMs: 1,+ }),+ ).resolves.toBe('key_1');+ expect(attempts).toBe(4);+ expect((await collection.indexes()).map((index) => index.name)).toContain('key_1');+ });++ test('surfaces the conflict once the peer-build deadline passes', async () => {+ const collection = rawCollection();+ let attempts = 0;+ collection.createIndex = async () => {+ attempts += 1;+ throw indexBuildInProgressError();+ };++ await expect(+ buildIndexWithRetry(() => collection.createIndex({ key: 1 }), 'key_1', {+ maxAttempts: 1,+ peerBuildPollMs: 1,+ peerBuildDeadlineMs: 20,+ }),+ ).rejects.toMatchObject({ code: INDEX_BUILD_ALREADY_IN_PROGRESS });+ expect(attempts).toBeGreaterThan(1);+ });++ test('still fails fast on errors that are not transient', async () => {+ const collection = rawCollection();+ let attempts = 0;+ collection.createIndex = async () => {+ attempts += 1;+ throw new mongoose.mongo.MongoServerError({+ ok: 0,+ code: 67,+ errmsg: 'CannotCreateIndex: bad index spec',+ });+ };++ await expect(+ buildIndexWithRetry(() => collection.createIndex({ key: 1 }), 'key_1', {+ baseDelayMs: 1,+ jitter: false,+ }),+ ).rejects.toThrow('bad index spec');+ expect(attempts).toBe(1);+ });+});++describe('retryWithBackoff option validation', () => {+ test.each([+ ['a NaN attempt count', { maxAttempts: NaN }],+ ['a NaN base delay', { baseDelayMs: NaN }],+ ['a NaN maximum delay', { maxDelayMs: NaN }],+ ])('rejects %s before running the operation', async (_shape, options) => {+ let attempts = 0;+ const operation = async () => {+ attempts += 1;+ };++ await expect(retryWithBackoff(operation, 'validation', options)).rejects.toThrow(+ 'Invalid options',+ );+ expect(attempts).toBe(0);+ });+});++describe('buildIndexWithRetry peer-build deadline', () => {+ test('reruns after a long admitted build reports a companion conflict', async () => {+ let attempts = 0;+ const build = async (): Promise<string> => {+ attempts += 1;+ if (attempts === 1) {+ await new Promise((resolve) => setTimeout(resolve, 30));+ throw indexBuildInProgressError();+ }+ return 'built';+ };++ await expect(+ buildIndexWithRetry(build, 'long-build', {+ maxAttempts: 1,+ peerBuildPollMs: 1,+ peerBuildDeadlineMs: 20,+ }),+ ).resolves.toBe('built');+ expect(attempts).toBe(2);+ });+});diff --git a/packages/data-schemas/src/utils/retry.ts b/packages/data-schemas/src/utils/retry.tsindex 0b42e7a753..7bf8c0284a 100644--- a/packages/data-schemas/src/utils/retry.ts+++ b/packages/data-schemas/src/utils/retry.ts@@ -28,7 +28,7 @@ export async function retryWithBackoff<T>(operation: () => Promise<T>,label: string,options: RetryOptions = {},-): Promise<T | undefined> {+): Promise<T> {const {maxAttempts = DEFAULT_OPTIONS.maxAttempts,baseDelayMs = DEFAULT_OPTIONS.baseDelayMs,@@ -37,7 +37,8 @@ export async function retryWithBackoff<T>(retryableErrors = DEFAULT_OPTIONS.retryableErrors,} = options;- if (maxAttempts < 1 || baseDelayMs < 0 || maxDelayMs < 0) {+ /** Negated comparisons, so a NaN option is rejected rather than skipping every attempt. */+ if (!(maxAttempts >= 1) || !(baseDelayMs >= 0) || !(maxDelayMs >= 0)) {throw new Error(`[retryWithBackoff] Invalid options: maxAttempts must be >= 1, delays must be non-negative`,);@@ -75,6 +76,9 @@ export async function retryWithBackoff<T>(await new Promise((resolve) => setTimeout(resolve, delayMs));}}+ throw new Error(+ `[retryWithBackoff] ${label} exhausted ${maxAttempts} attempt(s) without an outcome`,+ );}interface IndexedModel {@@ -96,7 +100,7 @@ const INDEX_BUILD_ALREADY_IN_PROGRESS = 40333;const INDEX_BUILD_IN_PROGRESS_MESSAGE = 'index build in progress';const DEFAULT_PEER_BUILD_POLL_MS = 5_000;-function isIndexBuildInProgress(error: unknown): boolean {+export function isIndexBuildInProgress(error: unknown): boolean {const candidate = error as { code?: number; message?: string } | null;if (candidate?.code === INDEX_BUILD_ALREADY_IN_PROGRESS) {return true;@@ -120,37 +124,65 @@ async function settleAutomaticIndexBuild(model: IndexedModel): Promise<void> {}/**- * Builds the model's indexes, waiting while another process (a peer replica- * booting the same release) holds the collection's single index build slot.- * A peer's build time is data-dependent, so the wait polls rather than backs- * off: the process cannot become ready without these indexes, and giving up- * only restarts it into the same wait. Every other error propagates at once.+ * Runs one index build, waiting while another process (a peer replica booting+ * the same release) holds the collection's single index build slot. A peer's+ * build time is data-dependent, so the wait polls rather than backs off: the+ * caller cannot become ready without the index, and giving up only restarts it+ * into the same wait. The deadline counts from the first conflict rather than+ * from the first attempt — a builder that admits one long build before+ * reporting another's conflict has not been waiting on anyone yet, so it always+ * gets a rerun. Every other error propagates at once.*/-async function buildIndexesWhenCollectionFree(- model: IndexedModel,+async function buildWhenCollectionFree<T>(+ build: () => Promise<T>,label: string,options: IndexBuildOptions,-): Promise<unknown> {+): Promise<T> {const pollMs = options.peerBuildPollMs ?? DEFAULT_PEER_BUILD_POLL_MS;- const startedAt = Date.now();+ let waitingSince: number | undefined;for (;;) {try {- return await model.createIndexes();+ return await build();} catch (error: unknown) {- const elapsedMs = Date.now() - startedAt;- const deadlineReached =- options.peerBuildDeadlineMs != null && elapsedMs >= options.peerBuildDeadlineMs;- if (!isIndexBuildInProgress(error) || deadlineReached) {+ if (!isIndexBuildInProgress(error)) {+ throw error;+ }+ if (waitingSince == null) {+ waitingSince = Date.now();+ }+ const elapsedMs = Date.now() - waitingSince;+ if (options.peerBuildDeadlineMs != null && elapsedMs >= options.peerBuildDeadlineMs) {throw error;}logger.warn(- `[createIndexesWithRetry] ${label} is waiting for another index build on the collection to finish (${Math.round(elapsedMs / 1000)}s elapsed, next attempt in ${pollMs}ms)`,+ `[buildIndexWithRetry] ${label} is waiting for another index build on the collection to finish (${Math.round(elapsedMs / 1000)}s elapsed, next attempt in ${pollMs}ms)`,);await new Promise((resolve) => setTimeout(resolve, pollMs));}}}+/**+ * Runs an index build with peer-build polling and transient-error retry, for a+ * caller that holds a raw driver collection or a third-party builder rather+ * than a Mongoose model. `build` is re-invoked until it settles, so it must be+ * idempotent — which every `createIndex` of an existing spec is.+ * Use this (or `createIndexesWithRetry`) instead of a raw `createIndex` on+ * FerretDB or DocumentDB; the static guard in `methods/documentdb.spec.ts`+ * rejects a raw call outside this module.+ */+export async function buildIndexWithRetry<T>(+ build: () => Promise<T>,+ label: string,+ options: IndexBuildOptions = {},+): Promise<T> {+ return await retryWithBackoff(+ () => buildWhenCollectionFree(build, label, options),+ label,+ options,+ );+}+/*** Creates all indexes for a Mongoose model with deadlock retry.* Use this instead of raw `model.createIndexes()` on FerretDB or DocumentDB.@@ -160,10 +192,9 @@ export async function createIndexesWithRetry(options: IndexBuildOptions = {},): Promise<void> {await settleAutomaticIndexBuild(model);- const label = `createIndexes(${model.modelName})`;- await retryWithBackoff(- () => buildIndexesWhenCollectionFree(model, label, options),- label,+ await buildIndexWithRetry(+ () => model.createIndexes(),+ `createIndexes(${model.modelName})`,options,);}