Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🍃 fix: Serialize Index Builds and the Agent Edge Cleanup for DocumentDB (#15670)

Danny Avila84851d86553b
diff --git a/packages/api/src/agents/checkpointer.integration.spec.ts b/packages/api/src/agents/checkpointer.integration.spec.ts
index a992822485..5a490a8a3e 100644
--- a/packages/api/src/agents/checkpointer.integration.spec.ts
+++ b/packages/api/src/agents/checkpointer.integration.spec.ts
@@ -16,6 +16,7 @@ import {
CheckpointTooLargeError,
LIBRECHAT_CHECKPOINT_NAMESPACE_KEY,
LIBRECHAT_EVENT_ACTOR_INVOCATION_KEY,
+ setupCheckpointIndexes,
__resetCheckpointerForTests,
} from './checkpointer';
@@ -1135,3 +1136,230 @@ describe('LazyMongoSaver checkpoint size guard (mongodb-memory-server integratio
expect(channels).toContain(NO_WRITES); // flushed, not dropped
});
});
+
+describe('setupCheckpointIndexes on a single-index-build engine (mongodb-memory-server)', () => {
+ const CHECKPOINTS = 'single_build_checkpoints';
+ const WRITES = 'single_build_checkpoint_writes';
+ /** Amazon DocumentDB: "Existing index build in progress on the same collection." */
+ const INDEX_BUILD_ALREADY_IN_PROGRESS = 40333;
+
+ const clientForSaver = () =>
+ mongoose.connection.getClient() as unknown as ConstructorParameters<
+ typeof MongoDBSaver
+ >[0]['client'];
+
+ /** Bound to Mongoose's database the way `buildMongoSaver` binds it, so the
+ * index assertions below read the collections the saver actually built. */
+ const makeSaver = () =>
+ new LazyMongoSaver({
+ client: clientForSaver(),
+ dbName: mongoose.connection.db?.databaseName,
+ checkpointCollectionName: CHECKPOINTS,
+ checkpointWritesCollectionName: WRITES,
+ ttl: 3600,
+ });
+
+ function indexBuildInProgressError(): Error {
+ return new mongoose.mongo.MongoServerError({
+ ok: 0,
+ code: INDEX_BUILD_ALREADY_IN_PROGRESS,
+ errmsg:
+ 'Existing index build in progress on the same collection. Collection is limited to a single index build at a time.',
+ });
+ }
+
+ interface IndexBuild {
+ collectionName: string;
+ indexName: string;
+ build: () => Promise<string>;
+ }
+
+ /**
+ * Routes every `createIndex` through `handler`, naming the index the way the
+ * server would. Patched on the driver prototype because the saver fetches a
+ * fresh `Collection` handle per call.
+ */
+ function patchCreateIndex(handler: (build: IndexBuild) => Promise<string>): () => void {
+ const prototype = mongoose.mongo.Collection.prototype;
+ const createIndex = prototype.createIndex;
+ prototype.createIndex = function (this: mongoose.mongo.Collection, spec, options) {
+ const indexName =
+ options?.name ??
+ Object.entries(spec)
+ .map(([field, direction]) => `${field}_${direction}`)
+ .join('_');
+ return handler({
+ collectionName: this.collectionName,
+ indexName,
+ build: () => createIndex.call(this, spec, options),
+ });
+ };
+ return () => {
+ prototype.createIndex = createIndex;
+ };
+ }
+
+ /**
+ * Turns the in-memory MongoDB into a single-index-build engine for the two
+ * checkpoint collections: a build that arrives while another is in flight on
+ * the same collection is rejected the way DocumentDB rejects it, instead of
+ * being serialized the way MongoDB does. Re-creating an index that already
+ * exists starts no build on either engine, so it always passes through.
+ */
+ function enforceSingleIndexBuild(): { rejected: () => number; restore: () => void } {
+ const inFlight = new Set<string>();
+ const built = new Set<string>();
+ let rejected = 0;
+ const restore = patchCreateIndex(async ({ collectionName, indexName, build }) => {
+ const key = `${collectionName}:${indexName}`;
+ if ((collectionName !== CHECKPOINTS && collectionName !== WRITES) || built.has(key)) {
+ return build();
+ }
+ if (inFlight.has(collectionName)) {
+ rejected += 1;
+ throw indexBuildInProgressError();
+ }
+ inFlight.add(collectionName);
+ try {
+ await new Promise((resolve) => setTimeout(resolve, 5));
+ const name = await build();
+ built.add(key);
+ return name;
+ } finally {
+ inFlight.delete(collectionName);
+ }
+ });
+ return { rejected: () => rejected, restore };
+ }
+
+ async function indexNames(collectionName: string): Promise<string[]> {
+ const indexes = await mongoose.connection.db!.collection(collectionName).indexes();
+ return indexes.map((index) => index.name ?? '');
+ }
+
+ it('models the engine: a raw setup() loses one build per collection', async () => {
+ const engine = enforceSingleIndexBuild();
+ try {
+ const errors = await makeSaver().setup();
+
+ expect(errors.map((error) => (error as { code?: number }).code)).toEqual([
+ INDEX_BUILD_ALREADY_IN_PROGRESS,
+ INDEX_BUILD_ALREADY_IN_PROGRESS,
+ ]);
+ expect(engine.rejected()).toBe(2);
+ } finally {
+ engine.restore();
+ }
+ });
+
+ it('re-runs setup() until every checkpoint index exists', async () => {
+ const engine = enforceSingleIndexBuild();
+ try {
+ await expect(
+ setupCheckpointIndexes(makeSaver(), { peerBuildPollMs: 1, peerBuildDeadlineMs: 5_000 }),
+ ).resolves.toEqual([]);
+
+ expect(engine.rejected()).toBe(2);
+ expect(await indexNames(CHECKPOINTS)).toEqual(
+ expect.arrayContaining(['thread_ns_checkpoint_idx', 'upserted_at_1']),
+ );
+ expect(await indexNames(WRITES)).toEqual(
+ expect.arrayContaining(['thread_ns_checkpoint_task_idx', 'upserted_at_1']),
+ );
+ } finally {
+ engine.restore();
+ }
+ });
+
+ it('returns errors that are not a concurrent build without re-running setup()', async () => {
+ const restore = patchCreateIndex(async ({ collectionName, build }) => {
+ if (collectionName !== WRITES) {
+ return build();
+ }
+ throw new mongoose.mongo.MongoServerError({
+ ok: 0,
+ code: 67,
+ errmsg: 'CannotCreateIndex: bad index spec',
+ });
+ });
+ try {
+ const saver = makeSaver();
+ const setup = jest.spyOn(saver, 'setup');
+
+ const errors = await setupCheckpointIndexes(saver, {
+ peerBuildPollMs: 1,
+ peerBuildDeadlineMs: 5_000,
+ });
+
+ expect(errors.map((error) => error.message)).toEqual([
+ 'CannotCreateIndex: bad index spec',
+ 'CannotCreateIndex: bad index spec',
+ ]);
+ expect(setup).toHaveBeenCalledTimes(1);
+ } finally {
+ restore();
+ }
+ });
+
+ it('proceeds with the conflict reported once the peer-build deadline passes', async () => {
+ const restore = patchCreateIndex(async ({ collectionName, build }) => {
+ if (collectionName !== CHECKPOINTS) {
+ return build();
+ }
+ throw indexBuildInProgressError();
+ });
+ try {
+ const errors = await setupCheckpointIndexes(makeSaver(), {
+ peerBuildPollMs: 1,
+ peerBuildDeadlineMs: 20,
+ });
+
+ expect(errors.map((error) => (error as { code?: number }).code)).toEqual([
+ INDEX_BUILD_ALREADY_IN_PROGRESS,
+ ]);
+ } finally {
+ restore();
+ }
+ });
+
+ it('keeps the failures reported beside a conflict that outlasts the deadline', async () => {
+ const restore = patchCreateIndex(async ({ collectionName }) => {
+ if (collectionName === CHECKPOINTS) {
+ throw indexBuildInProgressError();
+ }
+ throw new mongoose.mongo.MongoServerError({
+ ok: 0,
+ code: 67,
+ errmsg: 'CannotCreateIndex: bad index spec',
+ });
+ });
+ try {
+ const errors = await setupCheckpointIndexes(makeSaver(), {
+ peerBuildPollMs: 1,
+ peerBuildDeadlineMs: 20,
+ });
+
+ expect(errors.map((error) => (error as { code?: number }).code).sort()).toEqual([
+ INDEX_BUILD_ALREADY_IN_PROGRESS,
+ 67,
+ 67,
+ ]);
+ } finally {
+ restore();
+ }
+ });
+
+ it('propagates a setup() rejection that is not a build conflict', async () => {
+ /** The driver validates collection names server-side and `setup()` settles
+ * every build with `Promise.allSettled`, so there is no rejection path to
+ * trigger for real; a rejecting stand-in covers the contract that such a
+ * rejection reaches the caller's in-process fallback instead of being
+ * reported as an index error. */
+ const saver = makeSaver();
+ saver.setup = () => Promise.reject(new Error('client closed'));
+
+ await expect(setupCheckpointIndexes(saver, { peerBuildPollMs: 1 })).rejects.toThrow(
+ 'client closed',
+ );
+ });
+});
diff --git a/packages/api/src/agents/checkpointer.ts b/packages/api/src/agents/checkpointer.ts
index 63005be922..055867eaf1 100644
--- a/packages/api/src/agents/checkpointer.ts
+++ b/packages/api/src/agents/checkpointer.ts
@@ -1,7 +1,7 @@
import mongoose from 'mongoose';
-import { logger } from '@librechat/data-schemas';
import { INTERRUPT } from '@langchain/langgraph-checkpoint';
import { MongoDBSaver } from '@langchain/langgraph-checkpoint-mongodb';
+import { buildIndexWithRetry, isIndexBuildInProgress, logger } from '@librechat/data-schemas';
import type {
Checkpoint,
CheckpointListOptions,
@@ -11,6 +11,7 @@ import type {
} from '@langchain/langgraph-checkpoint';
import type { BaseMessage } from '@librechat/agents/langchain/messages';
import type { TCheckpointerConfig } from 'librechat-data-provider';
+import type { IndexBuildOptions } from '@librechat/data-schemas';
import type { RunnableConfig } from '@langchain/core/runnables';
/**
@@ -805,6 +806,48 @@ export async function captureAgentEventCheckpoint(
: null;
}
+/** Longest wait for a peer's checkpoint index build before the saver proceeds without it. */
+const CHECKPOINT_INDEX_BUILD_DEADLINE_MS = 120_000;
+
+/**
+ * `MongoDBSaver.setup()` starts the compound and TTL builds of each collection
+ * concurrently and reports failures instead of throwing. Amazon DocumentDB
+ * admits one index build per collection at a time, so there the second build of
+ * each pair is rejected (code 40333) and the saver would otherwise run with its
+ * TTL index missing — checkpoints then never expire. `setup()` is idempotent (an
+ * index that already exists is a no-op), so re-running it lets one more build
+ * through per pass until every index exists. Every other error is returned for
+ * the caller to log, exactly as `setup()` reports it — including those reported
+ * beside a conflict that outlasts the deadline. A rejection of `setup()` itself
+ * propagates, so the caller keeps its in-process fallback.
+ */
+export async function setupCheckpointIndexes(
+ saver: Pick<MongoDBSaver, 'setup'>,
+ options: IndexBuildOptions = {},
+): Promise<Error[]> {
+ let companions: Error[] = [];
+ try {
+ return await buildIndexWithRetry(
+ async () => {
+ const errors = await saver.setup();
+ const blocked = errors.find(isIndexBuildInProgress);
+ if (blocked == null) {
+ return errors;
+ }
+ companions = errors.filter((error) => !isIndexBuildInProgress(error));
+ throw blocked;
+ },
+ 'MongoDBSaver.setup()',
+ { peerBuildDeadlineMs: CHECKPOINT_INDEX_BUILD_DEADLINE_MS, ...options },
+ );
+ } catch (error) {
+ if (!isIndexBuildInProgress(error)) {
+ throw error;
+ }
+ return [...companions, error instanceof Error ? error : new Error(String(error))];
+ }
+}
+
async function buildMongoSaver(
resolved: ResolvedCheckpointerConfig,
): Promise<MongoDBSaver | undefined> {
@@ -829,7 +872,7 @@ async function buildMongoSaver(
// approval window, so a forgotten approval can never leak checkpoints forever.
ttl: resolved.ttlSeconds,
});
- const errors = await saver.setup();
+ const errors = await setupCheckpointIndexes(saver);
if (errors.length > 0) {
logger.warn(
'[checkpointer] MongoDBSaver.setup() reported errors (checkpoint indexes may be incomplete):',
diff --git a/packages/data-schemas/misc/documentdb/documentdb-compat.md b/packages/data-schemas/misc/documentdb/documentdb-compat.md
index 8152e360c2..a86d38a212 100644
--- a/packages/data-schemas/misc/documentdb/documentdb-compat.md
+++ b/packages/data-schemas/misc/documentdb/documentdb-compat.md
@@ -161,18 +161,69 @@ in the docs.
- **DocumentDB 8.0 pipeline-update acceptance** — 8.0 added `$set`/`$unset`
aggregation _stages_, but AWS never documents pipeline-form updates; the
harness probe answers this live.
-- **`collMod` is only "Partial"** on every version — avoid
- `Model.syncIndexes()` against DocumentDB (it may issue `collMod` beyond the
- documented `expireAfterSeconds`).
-- **Read-side aggregations** (3 files: `methods/prompt.ts`,
- `methods/aclEntry.ts`, `methods/agentCategory.ts`) were not audited
- stage-by-stage; no exotic stages (`$facet`, `$setWindowFields`,
- `$unionWith`, `$graphLookup`) are used anywhere.
+- **`collMod` is only "Partial"** on every version — `Model.syncIndexes()`
+ may issue `collMod` beyond the documented `expireAfterSeconds`, so the static
+ guard rejects it outright.
+- **Aggregation operators — now enforced, not observed.** 29 `aggregate`
+ calls across 11 files (`insights.ts`, `message.ts`, `mcpAuthority.ts`,
+ `agent.ts`, `aclEntry.ts`, `triggerDelivery.ts`, `queuedTurn.ts`,
+ `prompt.ts`, `conversation.ts`, `agentCategory.ts`,
+ `packages/api/src/code/lifecycle.ts`). An earlier inventory here named three
+ files because it was built from `.aggregate(`, which never matches the
+ generically typed `.aggregate<T>(` form the newer code uses. Every operator
+ in use (`$strLenBytes`, `$let`, `$map`, `$reduce`, `$regexMatch`, `$convert`,
+ `$anyElementTrue`, …) is in AWS's 5.0 supported list, and
+ `methods/documentdb.spec.ts` now rejects the full unsupported matrix,
+ `$set`/`$unset` at stage position, and the `$count` accumulator. Several
+ operators are 4.0+/5.0-only (`$expr`, `$switch`, `$convert`, `$regexMatch`,
+ array-form `$first`), which reinforces the 5.0+ recommendation.
- **No faithful local emulator exists.** The `documentdb-local` Docker image
is the PostgreSQL-based Linux Foundation project — AWS's own OSS blog
confirms "a different engine than the one used in Amazon DocumentDB." Live
regression testing must run against a real cluster.
+## Proven, fixed — edge cleanup pipeline update (every version)
+
+`removeAgentIdsFromEdges` (`methods/agent.ts`, since #14428) pruned deleted
+agent ids out of every graph's `edges` with an aggregation-pipeline update —
+the class fixed everywhere else in #15375. The static guard missed it because
+the pipeline reached `updateMany` as a function's return value rather than an
+array literal or an array-bound variable; the guard now follows calls to
+functions declared or annotated to return an array, and independently rejects
+`$set`/`$unset` at stage position. The rewrite reads each matching graph,
+prunes in code, and writes the result back behind a compare-and-set on the
+edges it read (`tenantSafeBulkWrite`, one round trip for every graph), so a
+concurrent edit is never overwritten — the atomicity the pipeline provided, at
+the cost of one extra read on the agent-delete path.
+
+## Proven, fixed — concurrent index builds (every version)
+
+DocumentDB admits one index build per collection at a time and rejects a
+second with code 40333 where MongoDB would serialize it. `utils/retry.ts`
+exists for exactly this and the boot builds go through it — but three callers
+bypassed it, and one sits on the agent hot path:
+
+- **`MongoDBSaver.setup()`** (`@langchain/langgraph-checkpoint-mongodb`) starts
+ the compound and TTL builds of each checkpoint collection in a single
+ `Promise.allSettled` and returns the rejections instead of throwing;
+ `buildMongoSaver` logged them and proceeded. On DocumentDB one build per
+ collection lost that race on every boot — the TTL index is pushed second,
+ making it the likely loser, so checkpoints accumulated without bound.
+ `setupCheckpointIndexes` now re-runs the idempotent `setup()` while any
+ rejection is a 40333; each pass admits one more build until all exist.
+- `methods/auditLog.ts` called `model.createIndexes()` raw before the chain's
+ first append; `migrations/mcpAuthorityIndexes.ts` and
+ `migrations/mcpServerNames.ts` called `collection.createIndex()` raw. All
+ three now go through the helpers (`buildIndexWithRetry` is the
+ raw-collection form of `createIndexesWithRetry`).
+
+`methods/documentdb.spec.ts` rejects any `createIndex` / `createIndexes` /
+`syncIndexes` / `ensureIndexes` outside `utils/retry.ts` unless it is the
+argument of `buildIndexWithRetry`, so the class cannot recur silently. The
+method sweep cannot see this class at all — it drives exported methods, and
+these builds happen inside a third-party `setup()` and in migrations — which
+is how it survived three audits.
+
## Regression strategy
1. **In-repo (CI today)**: the behavioral tests added in
@@ -189,14 +240,15 @@ in the docs.
## Support matrix and recommendation
-| Capability (LibreChat dependency) | 3.6 | 4.0 | 5.0 | 8.0 | Elastic |
-| --------------------------------------- | --- | --- | --- | --- | ------- |
-| Pipeline updates (**no longer used**) | ✗ | ✗ | ✗ | ? | ✗ |
-| Plain update operators (all writes now) | ✓ | ✓ | ✓ | ✓ | ✓ |
-| Unique indexes | ✓ | ✓ | ✓ | ✓ | ✗ |
-| Partial unique indexes (OAuth ids) | ✗ | ✗ | ✓ | ✓ | ✗ |
-| Transactions (runtime-probed) | ✗ | ✓ | ✓ | ✓ | ✗ |
-| TTL indexes | ✓ | ✓ | ✓ | ✓ | ✓ |
+| Capability (LibreChat dependency) | 3.6 | 4.0 | 5.0 | 8.0 | Elastic |
+| ---------------------------------------- | --- | --- | --- | --- | ------- |
+| Pipeline updates (**no longer used**) | ✗ | ✗ | ✗ | ? | ✗ |
+| Plain update operators (all writes now) | ✓ | ✓ | ✓ | ✓ | ✓ |
+| Unique indexes | ✓ | ✓ | ✓ | ✓ | ✗ |
+| Partial unique indexes (OAuth ids) | ✗ | ✗ | ✓ | ✓ | ✗ |
+| Transactions (runtime-probed) | ✗ | ✓ | ✓ | ✓ | ✗ |
+| TTL indexes | ✓ | ✓ | ✓ | ✓ | ✓ |
+| Concurrent index builds (now serialized) | ✗ | ✗ | ✗ | ✗ | ✗ |
**Recommendation**: support **DocumentDB 5.0+ instance-based** with
`retryWrites=false` documented as required. 4.0 functions with
diff --git a/packages/data-schemas/src/methods/agent.spec.ts b/packages/data-schemas/src/methods/agent.spec.ts
index a371438ea1..be07f8f2e9 100644
--- a/packages/data-schemas/src/methods/agent.spec.ts
+++ b/packages/data-schemas/src/methods/agent.spec.ts
@@ -20,8 +20,13 @@ import type {
Model,
} from 'mongoose';
import type { IAgent, IAclEntry, IUser, IAccessRole, CodeEnvironmentDocument } from '..';
+import {
+ createAgentMethods,
+ EDGE_CLEANUP_BATCH,
+ EDGE_CLEANUP_MAX_SWEEPS,
+ type AgentMethods,
+} from './agent';
import { withCodeEnvironmentReference } from './codeEnvironment';
-import { createAgentMethods, type AgentMethods } from './agent';
import { tenantStorage } from '~/config/tenantContext';
import { createAclEntryMethods } from './aclEntry';
import { createModels } from '~/models';
@@ -1265,6 +1270,207 @@ describe('Agent Methods', () => {
]);
});
+ test('keeps an edge added to the graph while the cleanup was running', async () => {
+ const authorId = new mongoose.Types.ObjectId();
+ const deletedAgentId = `agent_${uuidv4()}`;
+ const graphAgentId = `agent_${uuidv4()}`;
+ const sourceAgentId = `agent_${uuidv4()}`;
+ const targetAgentId = `agent_${uuidv4()}`;
+ const addedAgentId = `agent_${uuidv4()}`;
+ await createAgent({
+ id: deletedAgentId,
+ name: 'Agent To Delete',
+ provider: 'test',
+ model: 'test-model',
+ author: authorId,
+ });
+ await createAgent({
+ id: graphAgentId,
+ name: 'Agent Edited During Cleanup',
+ provider: 'test',
+ model: 'test-model',
+ author: authorId,
+ edges: [
+ { from: deletedAgentId, to: targetAgentId, edgeType: 'handoff' },
+ { from: sourceAgentId, to: targetAgentId, edgeType: 'handoff' },
+ ],
+ });
+ /** Lands a concurrent edit between the cleanup's read and its first write,
+ * through the driver so the cleanup's own compare-and-set is what must
+ * notice it. */
+ const prototype = mongoose.mongo.Collection.prototype;
+ const bulkWrite = prototype.bulkWrite;
+ let edited = false;
+ prototype.bulkWrite = async function (this: mongoose.mongo.Collection, operations, options) {
+ if (!edited && this.collectionName === 'agents') {
+ edited = true;
+ await Agent.updateOne(
+ { id: graphAgentId },
+ { $push: { edges: { from: sourceAgentId, to: addedAgentId, edgeType: 'handoff' } } },
+ );
+ }
+ return bulkWrite.call(this, operations, options);
+ };
+
+ try {
+ await deleteAgent({ id: deletedAgentId });
+ } finally {
+ prototype.bulkWrite = bulkWrite;
+ }
+
+ expect(edited).toBe(true);
+ const graphAgent = await getAgent({ id: graphAgentId });
+ expect(graphAgent!.edges).toEqual([
+ { from: sourceAgentId, to: targetAgentId, edgeType: 'handoff' },
+ { from: sourceAgentId, to: addedAgentId, edgeType: 'handoff' },
+ ]);
+ });
+
+ test('cleans every graph when the references span more than one page', async () => {
+ const authorId = new mongoose.Types.ObjectId();
+ const deletedAgentId = `agent_${uuidv4()}`;
+ const targetAgentId = `agent_${uuidv4()}`;
+ const graphCount = EDGE_CLEANUP_BATCH + 1;
+ await createAgent({
+ id: deletedAgentId,
+ name: 'Popular Agent',
+ provider: 'test',
+ model: 'test-model',
+ author: authorId,
+ });
+ await Agent.insertMany(
+ Array.from({ length: graphCount }, (_, index) => ({
+ id: `agent_${uuidv4()}`,
+ name: `Paged Graph ${index}`,
+ provider: 'test',
+ model: 'test-model',
+ author: authorId,
+ edges: [
+ { from: deletedAgentId, to: targetAgentId, edgeType: 'handoff' },
+ { from: targetAgentId, to: [deletedAgentId, targetAgentId], edgeType: 'direct' },
+ ],
+ })),
+ );
+
+ await deleteAgent({ id: deletedAgentId });
+
+ const graphs = await Agent.find({ name: /^Paged Graph / })
+ .select('edges')
+ .lean<Pick<IAgent, 'edges'>[]>();
+ expect(graphs).toHaveLength(graphCount);
+ const expectedEdges = [{ from: targetAgentId, to: [targetAgentId], edgeType: 'direct' }];
+ graphs.forEach((graph) => expect(graph.edges).toEqual(expectedEdges));
+ });
+
+ test('retries a compare-and-set miss the cursor has already passed', async () => {
+ const authorId = new mongoose.Types.ObjectId();
+ const deletedAgentId = `agent_${uuidv4()}`;
+ const targetAgentId = `agent_${uuidv4()}`;
+ const addedAgentId = `agent_${uuidv4()}`;
+ const graphCount = EDGE_CLEANUP_BATCH + 1;
+ await createAgent({
+ id: deletedAgentId,
+ name: 'Popular Agent',
+ provider: 'test',
+ model: 'test-model',
+ author: authorId,
+ });
+ const graphIds = Array.from({ length: graphCount }, () => `agent_${uuidv4()}`);
+ await Agent.insertMany(
+ graphIds.map((id, index) => ({
+ id,
+ name: `Cursor Graph ${index}`,
+ provider: 'test',
+ model: 'test-model',
+ author: authorId,
+ edges: [
+ { from: deletedAgentId, to: [deletedAgentId, targetAgentId], edgeType: 'direct' },
+ ],
+ })),
+ );
+ const [editedGraphId] = graphIds;
+ /** Lands a concurrent edit on a first-page graph between the cleanup's read
+ * and its first write, so that graph's compare-and-set misses while the
+ * cursor moves on past it. */
+ const prototype = mongoose.mongo.Collection.prototype;
+ const bulkWrite = prototype.bulkWrite;
+ let edited = false;
+ prototype.bulkWrite = async function (this: mongoose.mongo.Collection, operations, options) {
+ if (!edited && this.collectionName === 'agents') {
+ edited = true;
+ await Agent.updateOne(
+ { id: editedGraphId },
+ { $push: { edges: { from: targetAgentId, to: addedAgentId, edgeType: 'handoff' } } },
+ );
+ }
+ return bulkWrite.call(this, operations, options);
+ };
+
+ try {
+ await deleteAgent({ id: deletedAgentId });
+ } finally {
+ prototype.bulkWrite = bulkWrite;
+ }
+
+ expect(edited).toBe(true);
+ expect(
+ await Agent.countDocuments({
+ $or: [{ 'edges.from': deletedAgentId }, { 'edges.to': deletedAgentId }],
+ }),
+ ).toBe(0);
+ const editedGraph = await getAgent({ id: editedGraphId });
+ expect(editedGraph!.edges).toEqual([
+ { from: targetAgentId, to: addedAgentId, edgeType: 'handoff' },
+ ]);
+ });
+
+ test('gives up after a bounded number of sweeps when references keep being added', async () => {
+ const authorId = new mongoose.Types.ObjectId();
+ const deletedAgentId = `agent_${uuidv4()}`;
+ const graphAgentId = `agent_${uuidv4()}`;
+ const targetAgentId = `agent_${uuidv4()}`;
+ await createAgent({
+ id: deletedAgentId,
+ name: 'Agent To Delete',
+ provider: 'test',
+ model: 'test-model',
+ author: authorId,
+ });
+ await createAgent({
+ id: graphAgentId,
+ name: 'Graph That Keeps Referencing',
+ provider: 'test',
+ model: 'test-model',
+ author: authorId,
+ edges: [{ from: deletedAgentId, to: targetAgentId, edgeType: 'handoff' }],
+ });
+ /** Re-adds a reference AFTER every successful write, so every sweep makes
+ * progress and the stall bound never trips; only the sweep bound ends it. */
+ const prototype = mongoose.mongo.Collection.prototype;
+ const bulkWrite = prototype.bulkWrite;
+ let writes = 0;
+ prototype.bulkWrite = async function (this: mongoose.mongo.Collection, operations, options) {
+ const result = await bulkWrite.call(this, operations, options);
+ if (this.collectionName === 'agents') {
+ writes += 1;
+ await Agent.updateOne(
+ { id: graphAgentId },
+ { $push: { edges: { from: deletedAgentId, to: targetAgentId, edgeType: 'handoff' } } },
+ );
+ }
+ return result;
+ };
+
+ try {
+ await deleteAgent({ id: deletedAgentId });
+ } finally {
+ prototype.bulkWrite = bulkWrite;
+ }
+
+ expect(await getAgent({ id: deletedAgentId })).toBeNull();
+ expect(writes).toBeLessThanOrEqual(EDGE_CLEANUP_MAX_SWEEPS);
+ });
+
test('should remove agent from user favorites when agent is deleted', async () => {
const agentId = `agent_${uuidv4()}`;
const authorId = new mongoose.Types.ObjectId();
diff --git a/packages/data-schemas/src/methods/agent.ts b/packages/data-schemas/src/methods/agent.ts
index 4f2d392221..a11ba807ad 100644
--- a/packages/data-schemas/src/methods/agent.ts
+++ b/packages/data-schemas/src/methods/agent.ts
@@ -7,10 +7,11 @@ import {
actionDelimiter,
isActionTool,
} from 'librechat-data-provider';
-import type { FilterQuery, Model, PipelineStage, ProjectionType, Types } from 'mongoose';
+import type { FilterQuery, Model, ProjectionType, Types } from 'mongoose';
import type { AgentToolResources } from 'librechat-data-provider';
import type { IAgent, IAclEntry, ActionQuery } from '~/types';
import { withCodeEnvironmentReference } from './codeEnvironment';
+import { tenantSafeBulkWrite } from '~/utils/tenantBulkWrite';
import { filterExistingSkillIds } from './skill';
import logger from '~/config/winston';
@@ -45,71 +46,57 @@ const TOOL_RESOURCE_KEYS: ReadonlyArray<keyof AgentToolResources> = [
EToolResources.ocr,
];
-/** Builds an atomic update that prunes deleted IDs without discarding surviving edge members. */
-function createEdgeCleanupPipeline(agentIds: string[]): PipelineStage[] {
- const cleanEndpoint = (endpoint: string) => ({
- $cond: [
- { $isArray: endpoint },
- {
- $filter: {
- input: endpoint,
- as: 'agentId',
- cond: { $not: [{ $in: ['$$agentId', agentIds] }] },
- },
- },
- { $cond: [{ $in: [endpoint, agentIds] }, null, endpoint] },
- ],
- });
- const hasEndpoint = (endpoint: string) => ({
- $cond: [{ $isArray: endpoint }, { $gt: [{ $size: endpoint }, 0] }, { $ne: [endpoint, null] }],
+/** Graphs read per cleanup pass; bounds application memory the way the server-side update did. */
+export const EDGE_CLEANUP_BATCH = 200;
+/** Consecutive passes that may clean nothing before the loop gives up. */
+const EDGE_CLEANUP_STALLED_PASSES = 5;
+/** Sweeps from the top before the loop gives up on references that keep being added. */
+export const EDGE_CLEANUP_MAX_SWEEPS = 5;
+
+type AgentEdge = NonNullable<IAgent['edges']>[number];
+type EdgeEndpoint = AgentEdge['from'];
+
+/** An endpoint with `removed` taken out: a list loses those ids; a single id that is one becomes null. */
+function pruneEndpoint(
+ endpoint: EdgeEndpoint | null | undefined,
+ removed: Set<string>,
+): EdgeEndpoint | null {
+ if (Array.isArray(endpoint)) {
+ return endpoint.filter((id) => !removed.has(id));
+ }
+ return typeof endpoint === 'string' && removed.has(endpoint) ? null : (endpoint ?? null);
+}
+
+function hasEndpoint(endpoint: EdgeEndpoint | null): endpoint is EdgeEndpoint {
+ return Array.isArray(endpoint) ? endpoint.length > 0 : endpoint != null;
+}
+
+/** The edges that survive removing `removed`, with those ids pruned from their endpoints. */
+export function pruneEdges(edges: IAgent['edges'], removed: Set<string>): AgentEdge[] {
+ return (edges ?? []).flatMap((edge) => {
+ const from = pruneEndpoint(edge.from, removed);
+ const to = pruneEndpoint(edge.to, removed);
+ return hasEndpoint(from) && hasEndpoint(to) ? [{ ...edge, from, to }] : [];
});
+}
- return [
- {
- $set: {
- edges: {
- $filter: {
- input: {
- $map: {
- input: { $ifNull: ['$edges', []] },
- as: 'edge',
- in: {
- $let: {
- vars: {
- cleanedFrom: cleanEndpoint('$$edge.from'),
- cleanedTo: cleanEndpoint('$$edge.to'),
- },
- in: {
- $cond: [
- {
- $and: [hasEndpoint('$$cleanedFrom'), hasEndpoint('$$cleanedTo')],
- },
- {
- $mergeObjects: [
- '$$edge',
- {
- from: '$$cleanedFrom',
- to: '$$cleanedTo',
- },
- ],
- },
- null,
- ],
- },
- },
- },
- },
- },
- as: 'edge',
- cond: { $ne: ['$$edge', null] },
- },
- },
- },
- },
- ];
+interface GraphEdges {
+ _id: Types.ObjectId;
+ edges?: IAgent['edges'];
}
-/** Removes deleted agent references from active graphs in the requested tenant. */
+/**
+ * Removes deleted agent references from active graphs in the requested tenant.
+ * Graphs are read a page at a time behind an `_id` cursor, and each page's
+ * pruned edges are written back behind a compare-and-set on the edges that were
+ * read, so a concurrent edit is never overwritten. A graph whose edges changed
+ * underneath fails its compare-and-set and is left behind the cursor, so once
+ * the cursor is exhausted one more sweep from the top picks up every miss (and
+ * any reference added meanwhile); the cleanup ends when a sweep from the top
+ * finds nothing, and gives up after a bounded number of sweeps if references
+ * keep being added. This is the plain-operator form of what was an
+ * aggregation-pipeline update, which Amazon DocumentDB rejects.
+ */
async function removeAgentIdsFromEdges(
Agent: Model<IAgent>,
agentIds: string[],
@@ -118,14 +105,51 @@ async function removeAgentIdsFromEdges(
if (agentIds.length === 0) {
return;
}
-
- await Agent.updateMany(
- {
- ...(tenantId !== undefined ? { tenantId } : {}),
- $or: [{ 'edges.from': { $in: agentIds } }, { 'edges.to': { $in: agentIds } }],
- },
- createEdgeCleanupPipeline(agentIds),
- );
+ const filter: FilterQuery<IAgent> = {
+ ...(tenantId !== undefined ? { tenantId } : {}),
+ $or: [{ 'edges.from': { $in: agentIds } }, { 'edges.to': { $in: agentIds } }],
+ };
+ const removed = new Set(agentIds);
+ let stalledPasses = 0;
+ let sweeps = 0;
+ let after: Types.ObjectId | undefined;
+ for (;;) {
+ const graphs = await Agent.find(after == null ? filter : { ...filter, _id: { $gt: after } })
+ .sort({ _id: 1 })
+ .limit(EDGE_CLEANUP_BATCH)
+ .select('_id edges')
+ .lean<GraphEdges[]>();
+ if (graphs.length === 0) {
+ if (after == null) {
+ return;
+ }
+ sweeps += 1;
+ if (sweeps >= EDGE_CLEANUP_MAX_SWEEPS) {
+ throw new Error(
+ `[removeAgentIdsFromEdges] references kept being added during cleanup (${EDGE_CLEANUP_MAX_SWEEPS} sweeps)`,
+ );
+ }
+ after = undefined;
+ continue;
+ }
+ const result = await tenantSafeBulkWrite(
+ Agent,
+ graphs.map((graph) => ({
+ updateOne: {
+ filter: { _id: graph._id, edges: graph.edges },
+ update: { $set: { edges: pruneEdges(graph.edges, removed) } },
+ },
+ })),
+ { ordered: false },
+ );
+ stalledPasses = result.matchedCount === 0 ? stalledPasses + 1 : 0;
+ if (stalledPasses >= EDGE_CLEANUP_STALLED_PASSES) {
+ throw new Error(
+ `[removeAgentIdsFromEdges] graph edges kept changing during cleanup (${EDGE_CLEANUP_STALLED_PASSES} passes without progress)`,
+ );
+ }
+ after = graphs[graphs.length - 1]._id;
+ }
}
export interface AgentDeps {
diff --git a/packages/data-schemas/src/methods/auditLog.spec.ts b/packages/data-schemas/src/methods/auditLog.spec.ts
index aefdf34e81..9ae3262696 100644
--- a/packages/data-schemas/src/methods/auditLog.spec.ts
+++ b/packages/data-schemas/src/methods/auditLog.spec.ts
@@ -506,6 +506,32 @@ describe('auditLog methods', () => {
});
describe('append index safety', () => {
+ it('fails open within the index-build deadline while a peer holds the collection', async () => {
+ const prototype = mongoose.mongo.Collection.prototype;
+ const createIndex = prototype.createIndex;
+ const collectionName = AuditLog.collection.name;
+ prototype.createIndex = async function (this: mongoose.mongo.Collection, spec, options) {
+ if (this.collectionName !== collectionName) {
+ return createIndex.call(this, spec, options);
+ }
+ throw new mongoose.mongo.MongoServerError({
+ ok: 0,
+ code: 40333,
+ errmsg:
+ 'Existing index build in progress on the same collection. Collection is limited to a single index build at a time.',
+ });
+ };
+ try {
+ const freshMethods = createAuditLogMethods(mongoose, {
+ indexBuild: { peerBuildPollMs: 1, peerBuildDeadlineMs: 20 },
+ });
+
+ await expect(freshMethods.recordAuditEntry(baseInput())).resolves.toBeNull();
+ } finally {
+ prototype.createIndex = createIndex;
+ }
+ });
+
it('builds the unique seq index before appending (independent of autoIndex)', async () => {
const spy = jest.spyOn(AuditLog, 'createIndexes');
// a fresh methods instance has not yet memoized the index build
diff --git a/packages/data-schemas/src/methods/auditLog.ts b/packages/data-schemas/src/methods/auditLog.ts
index 0ab6a7dd51..d645b9c574 100644
--- a/packages/data-schemas/src/methods/auditLog.ts
+++ b/packages/data-schemas/src/methods/auditLog.ts
@@ -14,7 +14,9 @@ import type {
RecordAuditEntryOptions,
VerifyAuditChainOptions,
} from '~/types';
+import type { IndexBuildOptions } from '~/utils/retry';
import { GENESIS_HASH, PLATFORM_CHAIN_KEY } from '~/schema/auditLog';
+import { createIndexesWithRetry } from '~/utils/retry';
import { AUDIT_ACTION_CATEGORY } from '~/types/admin';
import logger from '~/config/winston';
@@ -290,7 +292,18 @@ function buildFilter(chainKey: string, filters: AuditLogFilters): FilterQuery<IA
return query;
}
-export function createAuditLogMethods(mongoose: typeof import('mongoose')): AuditLogMethods {
+/** Longest wait on the request path for a peer's index build before an append fails open. */
+const AUDIT_INDEX_BUILD_DEADLINE_MS = 10_000;
+
+export interface AuditLogMethodOptions {
+ /** Overrides for the append index build; tests shorten its polling and deadline. */
+ indexBuild?: IndexBuildOptions;
+}
+
+export function createAuditLogMethods(
+ mongoose: typeof import('mongoose'),
+ options: AuditLogMethodOptions = {},
+): AuditLogMethods {
function model(): Model<IAuditLog> {
return mongoose.models.AuditLog as Model<IAuditLog>;
}
@@ -301,18 +314,22 @@ export function createAuditLogMethods(mongoose: typeof import('mongoose')): Audi
* during the startup window before a background build finishes, two writers
* could insert the same `seq` with no duplicate-key error and silently fork
* the chain. Build the indexes once before the first append so serialization
- * never depends on a background build. Memoized; reset on failure so a later
- * write retries.
+ * never depends on a background build — letting Mongoose's own background
+ * build settle first, so the two never overlap on a single-build engine. The
+ * wait for a peer's build is bounded, because this runs on the request path:
+ * past the deadline the append fails open below instead of holding the
+ * request. Memoized; reset on failure so a later write retries.
*/
let indexPromise: Promise<unknown> | null = null;
function ensureIndexes(): Promise<unknown> {
if (!indexPromise) {
- indexPromise = model()
- .createIndexes()
- .catch((err) => {
- indexPromise = null;
- throw err;
- });
+ indexPromise = createIndexesWithRetry(model(), {
+ peerBuildDeadlineMs: AUDIT_INDEX_BUILD_DEADLINE_MS,
+ ...options.indexBuild,
+ }).catch((err) => {
+ indexPromise = null;
+ throw err;
+ });
}
return indexPromise;
}
diff --git a/packages/data-schemas/src/methods/documentdb.spec.ts b/packages/data-schemas/src/methods/documentdb.spec.ts
index 4cc139514d..e898016af0 100644
--- a/packages/data-schemas/src/methods/documentdb.spec.ts
+++ b/packages/data-schemas/src/methods/documentdb.spec.ts
@@ -15,9 +15,16 @@ import ts from 'typescript';
*
* The scan walks the TypeScript AST rather than source text, so it also
* catches a pipeline bound to a variable first, cast with `as`, or nested in a
- * `bulkWrite` operation's `update` property. It covers every backend workspace
+ * `bulkWrite` operation's `update` property. Besides constructs the engine
+ * rejects outright, it holds every index build to the retrying helpers in
+ * `utils/retry.ts`, because the engine admits one build per collection at a
+ * time and a concurrent second build fails silently in any caller that only
+ * logs. It covers every backend workspace
* that talks to MongoDB — this package, `packages/api`, and `api` — because the
* regression class is repo-wide and new backend code lands in `packages/api`.
+ * Dataflow is followed within a file only — a pipeline imported from another
+ * module is out of reach — and the method sweep and the live cluster run are
+ * the completeness backstops, not this guard.
* If a construct here becomes genuinely necessary, the fix is a compatible
* rewrite, not an exception list: `misc/documentdb/audit.documentdb.spec.ts`
* re-adjudicates any of this against a real cluster.
@@ -40,8 +47,123 @@ const UPDATE_METHODS = new Set([
'replaceOne',
]);
-/** `$$REMOVE`: `Feature not supported: $$REMOVE`. `$facet`: `Aggregation stage not supported`. */
-const FORBIDDEN_TOKENS = ['$$REMOVE', '$$CURRENT', '$facet', '$graphLookup', '$unionWith'];
+/**
+ * Every operator AWS documents as unsupported in the 5.0 column of "Supported
+ * MongoDB APIs, operations, and data types", grouped as that table groups them.
+ * The first two carry the exact server errors observed live — `$$REMOVE`:
+ * `Feature not supported: $$REMOVE`; `$facet`: `Aggregation stage not
+ * supported`. `$pow`, `$rand`, `$dateFromParts` and `$dateToParts` arrive in
+ * 5.0.1, after the 5.0.0 engine these verdicts were established on. `$set`,
+ * `$unset` and `$count` are legal in one position and rejected in another, so
+ * they have their own detectors below.
+ */
+const FORBIDDEN_TOKENS = [
+ '$$REMOVE',
+ '$$CURRENT',
+ // stages
+ '$facet',
+ '$graphLookup',
+ '$unionWith',
+ '$setWindowFields',
+ '$bucket',
+ '$bucketAuto',
+ '$merge',
+ '$replaceWith',
+ '$sortByCount',
+ '$vectorSearch',
+ '$listSearchIndexes',
+ '$planCacheStats',
+ '$listSessions',
+ '$listLocalSessions',
+ // accumulators
+ '$accumulator',
+ '$rank',
+ '$denseRank',
+ '$documentNumber',
+ '$shift',
+ '$derivative',
+ '$integral',
+ '$expMovingAvg',
+ '$covariancePop',
+ '$covarianceSamp',
+ '$stdDevPop',
+ '$stdDevSamp',
+ '$top',
+ '$topN',
+ '$bottom',
+ '$bottomN',
+ '$firstN',
+ '$lastN',
+ '$maxN',
+ '$minN',
+ '$median',
+ '$percentile',
+ // expressions
+ '$round',
+ '$trunc',
+ '$pow',
+ '$rand',
+ '$sortArray',
+ '$binarySize',
+ '$bsonSize',
+ '$dateTrunc',
+ '$dateFromParts',
+ '$dateToParts',
+ '$isNumber',
+ '$toUUID',
+ '$getField',
+ '$sampleRate',
+ '$sigmoid',
+ '$bitAnd',
+ '$bitNot',
+ '$bitOr',
+ '$bitXor',
+ '$tsIncrement',
+ '$tsSecond',
+ '$acos',
+ '$acosh',
+ '$asin',
+ '$asinh',
+ '$atan',
+ '$atan2',
+ '$atanh',
+ '$cos',
+ '$cosh',
+ '$sin',
+ '$sinh',
+ '$tan',
+ '$tanh',
+ '$degreesToRadians',
+ '$radiansToDegrees',
+ // query and cursor options
+ '$where',
+ 'allowDiskUse',
+ 'noCursorTimeout',
+];
+
+/** `$set` and `$unset` are update operators everywhere in this codebase and are
+ * supported as such; as pipeline STAGES (the `$addFields` / `$project` aliases)
+ * DocumentDB 5.0 rejects them. A stage is an object that is an element of an
+ * array, which no update-operator position is: an update document is a call
+ * argument or an `update` property, and a `bulkWrite` element keys on its
+ * operation name. */
+const ALIAS_STAGES = new Set(['$set', '$unset']);
+
+/** Index builds. The engine admits one build per collection at a time and
+ * rejects a second with code 40333 (`utils/retry.ts`), so every build must go
+ * through `createIndexesWithRetry` or `buildIndexWithRetry`, which poll for the
+ * slot. A raw call is legal only inside the helper module itself or as the
+ * argument of `buildIndexWithRetry`. */
+const INDEX_BUILD_METHODS = new Set([
+ 'createIndex',
+ 'createIndexes',
+ 'syncIndexes',
+ 'ensureIndexes',
+]);
+const INDEX_BUILD_HELPER = 'packages/data-schemas/src/utils/retry.ts';
+const INDEX_BUILD_WRAPPER = 'buildIndexWithRetry';
+/** Modules that name forbidden operators in order to reject them. */
+const OPERATOR_GUARDS = new Set(['packages/data-schemas/src/tenant/probe.ts']);
function collectSourceFiles(directory: string, found: string[] = []): string[] {
for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
@@ -78,25 +200,141 @@ function unwrapExpression(expression: ts.Expression): ts.Expression {
return current;
}
-/** Names of variables initialized with array literals, so an indirect
- * `const update = [...]; Model.updateOne(filter, update)` is still caught.
- * Scope-naive by design: a false positive here names a variable that holds an
- * array and is passed as an update, which deserves a look regardless. */
-function collectArrayVariableNames(sourceFile: ts.SourceFile): Set<string> {
- const names = new Set<string>();
+function isArrayType(type: ts.TypeNode | undefined): boolean {
+ if (type == null) {
+ return false;
+ }
+ if (ts.isArrayTypeNode(type)) {
+ return true;
+ }
+ return (
+ ts.isTypeReferenceNode(type) && ts.isIdentifier(type.typeName) && type.typeName.text === 'Array'
+ );
+}
+
+function isArrayExpression(expression: ts.Expression, arrayNames: Set<string>): boolean {
+ const unwrapped = unwrapExpression(expression);
+ return (
+ ts.isArrayLiteralExpression(unwrapped) ||
+ (ts.isIdentifier(unwrapped) && arrayNames.has(unwrapped.text))
+ );
+}
+
+/** Whether a function is declared to return an array, or returns one from its
+ * own body — an array literal, or a variable bound to one (nested functions
+ * are not descended into). */
+function returnsArray(fn: ts.FunctionLikeDeclaration, arrayNames: Set<string>): boolean {
+ if (isArrayType(fn.type)) {
+ return true;
+ }
+ if (fn.body == null) {
+ return false;
+ }
+ if (!ts.isBlock(fn.body)) {
+ return isArrayExpression(fn.body, arrayNames);
+ }
+ let found = false;
const visit = (node: ts.Node): void => {
+ if (found || ts.isFunctionLike(node)) {
+ return;
+ }
+ if (
+ ts.isReturnStatement(node) &&
+ node.expression != null &&
+ isArrayExpression(node.expression, arrayNames)
+ ) {
+ found = true;
+ return;
+ }
+ ts.forEachChild(node, visit);
+ };
+ ts.forEachChild(fn.body, visit);
+ return found;
+}
+
+function isArrayReturningFunction(expression: ts.Expression, arrayNames: Set<string>): boolean {
+ return (
+ (ts.isArrowFunction(expression) || ts.isFunctionExpression(expression)) &&
+ returnsArray(expression, arrayNames)
+ );
+}
+
+/** Names of variables initialized with array literals, then of functions and
+ * methods that return one — an array literal or one of those variables — so an
+ * indirect `const update = [...]; Model.updateOne(filter, update)`, a
+ * `Model.updateMany(filter, buildPipeline(ids))` or a `builder.pipeline()` is
+ * still caught. Scope-naive by design: a false positive here names something
+ * that holds an array and is passed as an update, which deserves a look
+ * regardless. */
+function collectArrayValuedNames(sourceFile: ts.SourceFile): Set<string> {
+ const names = new Set<string>();
+ const visitVariables = (node: ts.Node): void => {
+ if (
+ ts.isVariableDeclaration(node) &&
+ ts.isIdentifier(node.name) &&
+ (isArrayType(node.type) ||
+ (node.initializer != null &&
+ ts.isArrayLiteralExpression(unwrapExpression(node.initializer))))
+ ) {
+ names.add(node.name.text);
+ }
+ ts.forEachChild(node, visitVariables);
+ };
+ visitVariables(sourceFile);
+ const visitFunctions = (node: ts.Node): void => {
if (
ts.isVariableDeclaration(node) &&
ts.isIdentifier(node.name) &&
node.initializer != null &&
- ts.isArrayLiteralExpression(unwrapExpression(node.initializer))
+ isArrayReturningFunction(unwrapExpression(node.initializer), names)
) {
names.add(node.name.text);
}
+ if (
+ (ts.isFunctionDeclaration(node) || ts.isMethodDeclaration(node)) &&
+ node.name != null &&
+ ts.isIdentifier(node.name) &&
+ returnsArray(node, names)
+ ) {
+ names.add(node.name.text);
+ }
+ if (
+ ts.isPropertyAssignment(node) &&
+ ts.isIdentifier(node.name) &&
+ isArrayReturningFunction(unwrapExpression(node.initializer), names)
+ ) {
+ names.add(node.name.text);
+ }
+ ts.forEachChild(node, visitFunctions);
+ };
+ visitFunctions(sourceFile);
+ return names;
+}
+
+/** Object literals bound to variable names, so a stage held in a variable and
+ * pushed or listed later is still inspected. Scope-naive like the array names. */
+function collectObjectValuedNames(
+ sourceFile: ts.SourceFile,
+): Map<string, ts.ObjectLiteralExpression> {
+ const objects = new Map<string, ts.ObjectLiteralExpression>();
+ const visit = (node: ts.Node): void => {
+ if (ts.isVariableDeclaration(node) && ts.isIdentifier(node.name) && node.initializer != null) {
+ const initializer = unwrapExpression(node.initializer);
+ if (ts.isObjectLiteralExpression(initializer)) {
+ objects.set(node.name.text, initializer);
+ }
+ }
ts.forEachChild(node, visit);
};
visit(sourceFile);
- return names;
+ return objects;
+}
+
+function calleeName(call: ts.CallExpression): string | undefined {
+ if (ts.isIdentifier(call.expression)) {
+ return call.expression.text;
+ }
+ return ts.isPropertyAccessExpression(call.expression) ? call.expression.name.text : undefined;
}
function isArrayValued(expression: ts.Expression, arrayNames: Set<string>): boolean {
@@ -107,6 +345,10 @@ function isArrayValued(expression: ts.Expression, arrayNames: Set<string>): bool
if (ts.isIdentifier(unwrapped)) {
return arrayNames.has(unwrapped.text);
}
+ if (ts.isCallExpression(unwrapped)) {
+ const name = calleeName(unwrapped);
+ return name != null && arrayNames.has(name);
+ }
if (ts.isConditionalExpression(unwrapped)) {
return (
isArrayValued(unwrapped.whenTrue, arrayNames) ||
@@ -125,7 +367,7 @@ function offenseAt(sourceFile: ts.SourceFile, node: ts.Node, label: string): str
* whether passed directly to an update method or carried inside a `bulkWrite`
* operation's `update` property. */
function findPipelineUpdates(sourceFile: ts.SourceFile): string[] {
- const arrayNames = collectArrayVariableNames(sourceFile);
+ const arrayNames = collectArrayValuedNames(sourceFile);
const offenses: string[] = [];
const visit = (node: ts.Node): void => {
if (
@@ -159,11 +401,19 @@ function findPipelineUpdates(sourceFile: ts.SourceFile): string[] {
}
/** Reports forbidden operator tokens in string literals and property names,
- * ignoring prose — the rewrites explain themselves by naming the construct. */
+ * ignoring prose — the rewrites explain themselves by naming the construct —
+ * and type members, which never reach the engine (Mongoose documents declare
+ * a `$where` field). */
function findForbiddenTokens(sourceFile: ts.SourceFile): string[] {
+ if (OPERATOR_GUARDS.has(sourceFile.fileName)) {
+ return [];
+ }
const offenses: string[] = [];
const visit = (node: ts.Node): void => {
- if (ts.isStringLiteralLike(node) || ts.isIdentifier(node)) {
+ if (
+ ts.isStringLiteralLike(node) ||
+ (ts.isIdentifier(node) && !ts.isPropertySignature(node.parent))
+ ) {
for (const token of FORBIDDEN_TOKENS) {
if (node.text === token || node.text.startsWith(`${token}.`)) {
offenses.push(offenseAt(sourceFile, node, token));
@@ -212,10 +462,139 @@ function findMixedSelectStrings(sourceFile: ts.SourceFile): string[] {
return offenses;
}
+function propertyName(property: ts.ObjectLiteralElementLike): string | undefined {
+ const name = property.name;
+ if (name == null || !(ts.isIdentifier(name) || ts.isStringLiteral(name))) {
+ return undefined;
+ }
+ return name.text;
+}
+
+/** Array methods whose object arguments become elements of the receiver. */
+const ARRAY_APPENDERS = new Set(['push', 'unshift', 'concat']);
+
+function aliasStageOffenses(
+ sourceFile: ts.SourceFile,
+ element: ts.Expression,
+ objects: Map<string, ts.ObjectLiteralExpression>,
+): string[] {
+ const unwrapped = unwrapExpression(element);
+ if (ts.isConditionalExpression(unwrapped)) {
+ return [
+ ...aliasStageOffenses(sourceFile, unwrapped.whenTrue, objects),
+ ...aliasStageOffenses(sourceFile, unwrapped.whenFalse, objects),
+ ];
+ }
+ const stage = ts.isIdentifier(unwrapped) ? objects.get(unwrapped.text) : unwrapped;
+ if (stage == null || !ts.isObjectLiteralExpression(stage)) {
+ return [];
+ }
+ return stage.properties
+ .filter((property) => ALIAS_STAGES.has(propertyName(property) ?? ''))
+ .map((property) => offenseAt(sourceFile, property, `${propertyName(property)} stage`));
+}
+
+/** The expressions that become elements of an array: the elements of a literal,
+ * and the arguments of `push` / `unshift` / `concat`, which is how a pipeline
+ * is assembled dynamically. */
+function arrayElements(node: ts.Node): readonly ts.Expression[] {
+ if (ts.isArrayLiteralExpression(node)) {
+ return node.elements;
+ }
+ if (
+ ts.isCallExpression(node) &&
+ ts.isPropertyAccessExpression(node.expression) &&
+ ARRAY_APPENDERS.has(node.expression.name.text)
+ ) {
+ return node.arguments;
+ }
+ return [];
+}
+
+/** Reports `$set` / `$unset` objects in element position — inside an array
+ * literal or appended to one, on either side of a conditional — wherever the
+ * pipeline is assembled. */
+function findAliasStages(sourceFile: ts.SourceFile): string[] {
+ const objects = collectObjectValuedNames(sourceFile);
+ const offenses: string[] = [];
+ const visit = (node: ts.Node): void => {
+ offenses.push(
+ ...arrayElements(node).flatMap((element) => aliasStageOffenses(sourceFile, element, objects)),
+ );
+ ts.forEachChild(node, visit);
+ };
+ visit(sourceFile);
+ return offenses;
+}
+
+/** `$count` is a supported STAGE (`{ $count: 'total' }`) but an unsupported
+ * ACCUMULATOR (`{ n: { $count: {} } }`) on 5.0; the two differ by value shape. */
+function findAccumulatorCounts(sourceFile: ts.SourceFile): string[] {
+ const offenses: string[] = [];
+ const visit = (node: ts.Node): void => {
+ if (
+ ts.isPropertyAssignment(node) &&
+ propertyName(node) === '$count' &&
+ ts.isObjectLiteralExpression(unwrapExpression(node.initializer))
+ ) {
+ offenses.push(offenseAt(sourceFile, node, '$count accumulator'));
+ }
+ ts.forEachChild(node, visit);
+ };
+ visit(sourceFile);
+ return offenses;
+}
+
+function isWrappedIndexBuild(node: ts.Node): boolean {
+ for (let current = node.parent; current != null; current = current.parent) {
+ if (
+ ts.isCallExpression(current) &&
+ ts.isIdentifier(current.expression) &&
+ current.expression.text === INDEX_BUILD_WRAPPER
+ ) {
+ return true;
+ }
+ }
+ return false;
+}
+
+/** Meilisearch's client shares the `createIndex` name; its options carry a
+ * `primaryKey`, which no MongoDB index option does. */
+function isMeilisearchIndexCall(call: ts.CallExpression): boolean {
+ const options = call.arguments[1] == null ? undefined : unwrapExpression(call.arguments[1]);
+ return (
+ options != null &&
+ ts.isObjectLiteralExpression(options) &&
+ options.properties.some((property) => propertyName(property) === 'primaryKey')
+ );
+}
+
+/** Reports every index build that bypasses the retrying helpers. */
+function findRawIndexBuilds(sourceFile: ts.SourceFile): string[] {
+ if (sourceFile.fileName === INDEX_BUILD_HELPER) {
+ return [];
+ }
+ const offenses: string[] = [];
+ const visit = (node: ts.Node): void => {
+ if (
+ ts.isCallExpression(node) &&
+ ts.isPropertyAccessExpression(node.expression) &&
+ INDEX_BUILD_METHODS.has(node.expression.name.text) &&
+ !isWrappedIndexBuild(node) &&
+ !isMeilisearchIndexCall(node)
+ ) {
+ offenses.push(offenseAt(sourceFile, node, `raw ${node.expression.name.text}`));
+ }
+ ts.forEachChild(node, visit);
+ };
+ visit(sourceFile);
+ return offenses;
+}
+
describe('Amazon DocumentDB compatibility', () => {
- /** Each file is read and parsed once; both detectors walk the same tree. */
+ /** Each file is read and parsed once; every detector walks the same tree. */
const parsedSources = SCAN_ROOTS.flatMap((root) => collectSourceFiles(root)).map((file) =>
- parse(path.relative(REPO_ROOT, file), fs.readFileSync(file, 'utf8')),
+ parse(path.relative(REPO_ROOT, file).split(path.sep).join('/'), fs.readFileSync(file, 'utf8')),
);
it('scans the backend workspaces', () => {
@@ -234,6 +613,18 @@ describe('Amazon DocumentDB compatibility', () => {
expect(parsedSources.flatMap(findMixedSelectStrings)).toEqual([]);
});
+ it('uses no $set or $unset pipeline stages', () => {
+ expect(parsedSources.flatMap(findAliasStages)).toEqual([]);
+ });
+
+ it('uses no $count accumulators', () => {
+ expect(parsedSources.flatMap(findAccumulatorCounts)).toEqual([]);
+ });
+
+ it('builds every index through the retrying helpers', () => {
+ expect(parsedSources.flatMap(findRawIndexBuilds)).toEqual([]);
+ });
+
/** A guard that cannot fail protects nothing, so every shape the detectors
* exist to catch is proven against fixtures here rather than by trusting a
* one-time manual injection. */
@@ -257,6 +648,34 @@ describe('Amazon DocumentDB compatibility', () => {
'bulkWrite indirect payload',
`const update = [{ $set: { a: 1 } }];\nawait Model.bulkWrite([{ updateMany: { filter, update } }]);`,
],
+ [
+ 'pipeline returned by a declared function',
+ `function pipeline(ids: string[]): PipelineStage[] {\n return [{ $set: { ids } }];\n}\nModel.updateMany(filter, pipeline(ids));`,
+ ],
+ [
+ 'pipeline returned by an arrow with an expression body',
+ `const pipeline = (ids: string[]) => [{ $set: { ids } }];\nModel.updateOne(filter, pipeline(ids));`,
+ ],
+ [
+ 'pipeline returned by an untyped function body',
+ `function pipeline() {\n const stage = { $set: { a: 1 } };\n return [stage] as PipelineStage[];\n}\nModel.findOneAndUpdate(filter, pipeline());`,
+ ],
+ [
+ 'pipeline returned by a class method',
+ `class Builder {\n pipeline(): PipelineStage[] {\n return [{ $addFields: { a: 1 } }];\n }\n}\nModel.updateMany(filter, new Builder().pipeline());`,
+ ],
+ [
+ 'pipeline returned by an object method',
+ `const builder = { pipeline: () => [{ $addFields: { a: 1 } }] };\nModel.updateOne(filter, builder.pipeline());`,
+ ],
+ [
+ 'pipeline returned through a local variable',
+ `function pipeline() {\n const stages = [{ $addFields: { a: 1 } }];\n return stages;\n}\nModel.updateMany(filter, pipeline());`,
+ ],
+ [
+ 'annotated variable with a builder initializer',
+ `const update: PipelineStage[] = importedBuilder();\nModel.updateMany(filter, update);`,
+ ],
])('flags a pipeline update: %s', (_shape, source) => {
expect(findPipelineUpdates(parse('fixture.ts', source))).not.toEqual([]);
});
@@ -268,6 +687,10 @@ describe('Amazon DocumentDB compatibility', () => {
`await Model.bulkWrite([{ updateOne: { filter, update: { $set: { a: 1 } } } }]);`,
],
['unrelated array variable', `const stages = [{ $match: {} }];\nModel.aggregate(stages);`],
+ [
+ 'update document returned by a function',
+ `function update() {\n return { $set: { a: 1 } };\n}\nModel.updateOne(filter, update());`,
+ ],
])('accepts a supported shape: %s', (_shape, source) => {
expect(findPipelineUpdates(parse('fixture.ts', source))).toEqual([]);
});
@@ -289,16 +712,96 @@ describe('Amazon DocumentDB compatibility', () => {
expect(findMixedSelectStrings(parse('fixture.ts', source))).toEqual([]);
});
- it('flags forbidden operators in code but not in prose', () => {
+ it('flags forbidden operators in code but not in prose or type members', () => {
expect(
findForbiddenTokens(parse('fixture.ts', `const projection = { x: '$$REMOVE' };`)),
).not.toEqual([]);
expect(
findForbiddenTokens(parse('fixture.ts', `pipeline.push({ $facet: { rows: [] } });`)),
).not.toEqual([]);
+ expect(
+ findForbiddenTokens(parse('fixture.ts', `const stage = { $setWindowFields: {} };`)),
+ ).not.toEqual([]);
expect(
findForbiddenTokens(parse('fixture.ts', `/** $$REMOVE and $facet are unsupported. */`)),
).toEqual([]);
+ expect(
+ findForbiddenTokens(
+ parse('fixture.ts', `interface Doc { $where: Record<string, unknown> }`),
+ ),
+ ).toEqual([]);
+ });
+
+ it.each([
+ ['stage after a match', `Model.aggregate([{ $match: {} }, { $set: { a: 1 } }]);`],
+ ['unset stage held in a variable', `const stages = [{ $unset: 'a' }];`],
+ ['stage appended to a spread scope', `Model.aggregate([...scope, { $set: { a: 1 } }]);`],
+ [
+ 'stage pushed onto a pipeline',
+ `const stages = [];\nstages.push({ $match: {} }, { $set: { a: 1 } });\nModel.aggregate(stages);`,
+ ],
+ ['stage prepended with unshift', `stages.unshift({ $unset: 'a' });`],
+ ['stage concatenated onto a base', `const stages = base.concat({ $set: { a: 1 } });`],
+ [
+ 'stage chosen by a conditional element',
+ `Model.aggregate([...scope, flag ? { $set: { a: 1 } } : { $addFields: { a: 1 } }]);`,
+ ],
+ [
+ 'stage held in a variable and pushed',
+ `const stage = { $set: { a: 1 } };\nstages.push(stage);`,
+ ],
+ [
+ 'stage held in a variable and listed',
+ `const stage = { $unset: 'a' };\nconst stages = [stage];`,
+ ],
+ ])('flags an alias stage: %s', (_shape, source) => {
+ expect(findAliasStages(parse('fixture.ts', source))).not.toEqual([]);
+ });
+
+ it.each([
+ ['classic update operator', `Model.updateOne(filter, { $set: { a: 1 } });`],
+ [
+ 'bulkWrite operation payload',
+ `await Model.bulkWrite([{ updateOne: { filter, update: { $set: { a: 1 } } } }]);`,
+ ],
+ ['addFields stage', `Model.aggregate([{ $addFields: { a: 1 } }]);`],
+ [
+ 'bulk operation pushed onto an operations list',
+ `const operations = [];\noperations.push({ updateOne: { filter, update: { $set: { a: 1 } } } });`,
+ ],
+ ])('accepts a supported $set position: %s', (_shape, source) => {
+ expect(findAliasStages(parse('fixture.ts', source))).toEqual([]);
+ });
+
+ it('flags a $count accumulator but not the $count stage', () => {
+ expect(
+ findAccumulatorCounts(
+ parse('fixture.ts', `Model.aggregate([{ $group: { _id: null, n: { $count: {} } } }]);`),
+ ),
+ ).not.toEqual([]);
+ expect(
+ findAccumulatorCounts(parse('fixture.ts', `Model.aggregate([{ $count: 'total' }]);`)),
+ ).toEqual([]);
+ });
+
+ it.each([
+ ['raw createIndex', `await collection.createIndex({ a: 1 });`],
+ ['raw createIndex with options', `await collection.createIndex({ a: 1 }, { unique: true });`],
+ ['raw createIndexes', `await Model.createIndexes();`],
+ ['raw syncIndexes', `await Model.syncIndexes();`],
+ ])('flags an unguarded index build: %s', (_shape, source) => {
+ expect(findRawIndexBuilds(parse('fixture.ts', source))).not.toEqual([]);
+ });
+
+ it.each([
+ [
+ 'a build wrapped by the helper',
+ `await buildIndexWithRetry(() => collection.createIndex({ a: 1 }), 'a_1');`,
+ ],
+ ['the model helper', `await createIndexesWithRetry(Model);`],
+ ['the Meilisearch client', `await client.createIndex('messages', { primaryKey: 'id' });`],
+ ])('accepts a guarded index build: %s', (_shape, source) => {
+ expect(findRawIndexBuilds(parse('fixture.ts', source))).toEqual([]);
});
});
});
diff --git a/packages/data-schemas/src/migrations/mcpAuthorityIndexes.ts b/packages/data-schemas/src/migrations/mcpAuthorityIndexes.ts
index 4062dc691c..74b08a52bf 100644
--- a/packages/data-schemas/src/migrations/mcpAuthorityIndexes.ts
+++ b/packages/data-schemas/src/migrations/mcpAuthorityIndexes.ts
@@ -1,5 +1,6 @@
import type { IndexSpecification } from 'mongodb';
import type { Connection } from 'mongoose';
+import { buildIndexWithRetry } from '~/utils/retry';
interface AuthorityIndexDefinition {
collection: string;
@@ -36,9 +37,11 @@ export async function createMCPAuthorityLookupIndexes(
): Promise<readonly string[]> {
const created: string[] = [];
for (const definition of AUTHORITY_INDEXES) {
- const name = await connection
- .db!.collection(definition.collection)
- .createIndex(definition.keys, { name: definition.name });
+ const collection = connection.db!.collection(definition.collection);
+ const name = await buildIndexWithRetry(
+ () => collection.createIndex(definition.keys, { name: definition.name }),
+ `createIndex(${definition.collection}.${definition.name})`,
+ );
created.push(name);
}
return created;
diff --git a/packages/data-schemas/src/migrations/mcpServerNames.ts b/packages/data-schemas/src/migrations/mcpServerNames.ts
index 8be0f51b45..c8f47a1120 100644
--- a/packages/data-schemas/src/migrations/mcpServerNames.ts
+++ b/packages/data-schemas/src/migrations/mcpServerNames.ts
@@ -1,5 +1,6 @@
import { normalizeServerName } from 'librechat-data-provider';
import type { Connection, Types } from 'mongoose';
+import { buildIndexWithRetry } from '~/utils/retry';
interface MCPServerNameRow {
_id: Types.ObjectId;
@@ -96,13 +97,17 @@ export async function backfillMCPServerNormalizedNames(
// eslint-disable-next-line no-restricted-syntax -- offline all-tenant migration intentionally bypasses request tenant scoping
await collection.bulkWrite(updates, { ordered: true });
}
- await collection.createIndex(
- { normalizedServerName: 1, tenantId: 1 },
- {
- name: 'normalizedServerName_1_tenantId_1',
- unique: true,
- partialFilterExpression: { normalizedServerName: { $exists: true } },
- },
+ await buildIndexWithRetry(
+ () =>
+ collection.createIndex(
+ { normalizedServerName: 1, tenantId: 1 },
+ {
+ name: 'normalizedServerName_1_tenantId_1',
+ unique: true,
+ partialFilterExpression: { normalizedServerName: { $exists: true } },
+ },
+ ),
+ 'createIndex(normalizedServerName_1_tenantId_1)',
);
return { scanned, updated };
}
diff --git a/packages/data-schemas/src/utils/index.ts b/packages/data-schemas/src/utils/index.ts
index 4adfa3488b..fecdfcc7ed 100644
--- a/packages/data-schemas/src/utils/index.ts
+++ b/packages/data-schemas/src/utils/index.ts
@@ -8,3 +8,5 @@ export * from './objectId';
export * from './yaml';
export * from './stripUIResourceMarkers';
export * from './fading';
+export { buildIndexWithRetry, createIndexesWithRetry, isIndexBuildInProgress } from './retry';
+export type { IndexBuildOptions } from './retry';
diff --git a/packages/data-schemas/src/utils/retry.spec.ts b/packages/data-schemas/src/utils/retry.spec.ts
index a7dbb257c6..6bc33ddbb5 100644
--- a/packages/data-schemas/src/utils/retry.spec.ts
+++ b/packages/data-schemas/src/utils/retry.spec.ts
@@ -2,8 +2,8 @@ import mongoose from 'mongoose';
import { MongoMemoryServer } from 'mongodb-memory-server';
import type { Model } from 'mongoose';
import type { IAgentTriggerDeliveryDocument } from '~/types/triggerDelivery';
+import { buildIndexWithRetry, createIndexesWithRetry, retryWithBackoff } from './retry';
import triggerDeliverySchema from '~/schema/triggerDelivery';
-import { createIndexesWithRetry } from './retry';
const DB_SETUP_TIMEOUT_MS = 60_000;
/** Amazon DocumentDB: "Existing index build in progress on the same collection." */
@@ -154,3 +154,114 @@ describe('createIndexesWithRetry on a single-index-build engine', () => {
expect(attempts).toBe(triggerDeliverySchema.indexes().length);
});
});
+
+describe('buildIndexWithRetry on a raw driver collection', () => {
+ function rawCollection() {
+ modelCounter += 1;
+ return mongoose.connection.db!.collection(`retry_spec_raw_${modelCounter}`);
+ }
+
+ test('keeps polling while another replica holds the collection, then builds', async () => {
+ const collection = rawCollection();
+ const createIndex = collection.createIndex.bind(collection);
+ let remainingRejections = 3;
+ let attempts = 0;
+ collection.createIndex = async (fields, options) => {
+ attempts += 1;
+ if (remainingRejections > 0) {
+ remainingRejections -= 1;
+ throw indexBuildInProgressError();
+ }
+ return createIndex(fields, options);
+ };
+
+ await expect(
+ buildIndexWithRetry(() => collection.createIndex({ key: 1 }, { name: 'key_1' }), 'key_1', {
+ maxAttempts: 1,
+ peerBuildPollMs: 1,
+ }),
+ ).resolves.toBe('key_1');
+ expect(attempts).toBe(4);
+ expect((await collection.indexes()).map((index) => index.name)).toContain('key_1');
+ });
+
+ test('surfaces the conflict once the peer-build deadline passes', async () => {
+ const collection = rawCollection();
+ let attempts = 0;
+ collection.createIndex = async () => {
+ attempts += 1;
+ throw indexBuildInProgressError();
+ };
+
+ await expect(
+ buildIndexWithRetry(() => collection.createIndex({ key: 1 }), 'key_1', {
+ maxAttempts: 1,
+ peerBuildPollMs: 1,
+ peerBuildDeadlineMs: 20,
+ }),
+ ).rejects.toMatchObject({ code: INDEX_BUILD_ALREADY_IN_PROGRESS });
+ expect(attempts).toBeGreaterThan(1);
+ });
+
+ test('still fails fast on errors that are not transient', async () => {
+ const collection = rawCollection();
+ let attempts = 0;
+ collection.createIndex = async () => {
+ attempts += 1;
+ throw new mongoose.mongo.MongoServerError({
+ ok: 0,
+ code: 67,
+ errmsg: 'CannotCreateIndex: bad index spec',
+ });
+ };
+
+ await expect(
+ buildIndexWithRetry(() => collection.createIndex({ key: 1 }), 'key_1', {
+ baseDelayMs: 1,
+ jitter: false,
+ }),
+ ).rejects.toThrow('bad index spec');
+ expect(attempts).toBe(1);
+ });
+});
+
+describe('retryWithBackoff option validation', () => {
+ test.each([
+ ['a NaN attempt count', { maxAttempts: NaN }],
+ ['a NaN base delay', { baseDelayMs: NaN }],
+ ['a NaN maximum delay', { maxDelayMs: NaN }],
+ ])('rejects %s before running the operation', async (_shape, options) => {
+ let attempts = 0;
+ const operation = async () => {
+ attempts += 1;
+ };
+
+ await expect(retryWithBackoff(operation, 'validation', options)).rejects.toThrow(
+ 'Invalid options',
+ );
+ expect(attempts).toBe(0);
+ });
+});
+
+describe('buildIndexWithRetry peer-build deadline', () => {
+ test('reruns after a long admitted build reports a companion conflict', async () => {
+ let attempts = 0;
+ const build = async (): Promise<string> => {
+ attempts += 1;
+ if (attempts === 1) {
+ await new Promise((resolve) => setTimeout(resolve, 30));
+ throw indexBuildInProgressError();
+ }
+ return 'built';
+ };
+
+ await expect(
+ buildIndexWithRetry(build, 'long-build', {
+ maxAttempts: 1,
+ peerBuildPollMs: 1,
+ peerBuildDeadlineMs: 20,
+ }),
+ ).resolves.toBe('built');
+ expect(attempts).toBe(2);
+ });
+});
diff --git a/packages/data-schemas/src/utils/retry.ts b/packages/data-schemas/src/utils/retry.ts
index 0b42e7a753..7bf8c0284a 100644
--- a/packages/data-schemas/src/utils/retry.ts
+++ b/packages/data-schemas/src/utils/retry.ts
@@ -28,7 +28,7 @@ export async function retryWithBackoff<T>(
operation: () => Promise<T>,
label: string,
options: RetryOptions = {},
-): Promise<T | undefined> {
+): Promise<T> {
const {
maxAttempts = DEFAULT_OPTIONS.maxAttempts,
baseDelayMs = DEFAULT_OPTIONS.baseDelayMs,
@@ -37,7 +37,8 @@ export async function retryWithBackoff<T>(
retryableErrors = DEFAULT_OPTIONS.retryableErrors,
} = options;
- if (maxAttempts < 1 || baseDelayMs < 0 || maxDelayMs < 0) {
+ /** Negated comparisons, so a NaN option is rejected rather than skipping every attempt. */
+ if (!(maxAttempts >= 1) || !(baseDelayMs >= 0) || !(maxDelayMs >= 0)) {
throw new Error(
`[retryWithBackoff] Invalid options: maxAttempts must be >= 1, delays must be non-negative`,
);
@@ -75,6 +76,9 @@ export async function retryWithBackoff<T>(
await new Promise((resolve) => setTimeout(resolve, delayMs));
}
}
+ throw new Error(
+ `[retryWithBackoff] ${label} exhausted ${maxAttempts} attempt(s) without an outcome`,
+ );
}
interface IndexedModel {
@@ -96,7 +100,7 @@ const INDEX_BUILD_ALREADY_IN_PROGRESS = 40333;
const INDEX_BUILD_IN_PROGRESS_MESSAGE = 'index build in progress';
const DEFAULT_PEER_BUILD_POLL_MS = 5_000;
-function isIndexBuildInProgress(error: unknown): boolean {
+export function isIndexBuildInProgress(error: unknown): boolean {
const candidate = error as { code?: number; message?: string } | null;
if (candidate?.code === INDEX_BUILD_ALREADY_IN_PROGRESS) {
return true;
@@ -120,37 +124,65 @@ async function settleAutomaticIndexBuild(model: IndexedModel): Promise<void> {
}
/**
- * Builds the model's indexes, waiting while another process (a peer replica
- * booting the same release) holds the collection's single index build slot.
- * A peer's build time is data-dependent, so the wait polls rather than backs
- * off: the process cannot become ready without these indexes, and giving up
- * only restarts it into the same wait. Every other error propagates at once.
+ * Runs one index build, waiting while another process (a peer replica booting
+ * the same release) holds the collection's single index build slot. A peer's
+ * build time is data-dependent, so the wait polls rather than backs off: the
+ * caller cannot become ready without the index, and giving up only restarts it
+ * into the same wait. The deadline counts from the first conflict rather than
+ * from the first attempt — a builder that admits one long build before
+ * reporting another's conflict has not been waiting on anyone yet, so it always
+ * gets a rerun. Every other error propagates at once.
*/
-async function buildIndexesWhenCollectionFree(
- model: IndexedModel,
+async function buildWhenCollectionFree<T>(
+ build: () => Promise<T>,
label: string,
options: IndexBuildOptions,
-): Promise<unknown> {
+): Promise<T> {
const pollMs = options.peerBuildPollMs ?? DEFAULT_PEER_BUILD_POLL_MS;
- const startedAt = Date.now();
+ let waitingSince: number | undefined;
for (;;) {
try {
- return await model.createIndexes();
+ return await build();
} catch (error: unknown) {
- const elapsedMs = Date.now() - startedAt;
- const deadlineReached =
- options.peerBuildDeadlineMs != null && elapsedMs >= options.peerBuildDeadlineMs;
- if (!isIndexBuildInProgress(error) || deadlineReached) {
+ if (!isIndexBuildInProgress(error)) {
+ throw error;
+ }
+ if (waitingSince == null) {
+ waitingSince = Date.now();
+ }
+ const elapsedMs = Date.now() - waitingSince;
+ if (options.peerBuildDeadlineMs != null && elapsedMs >= options.peerBuildDeadlineMs) {
throw error;
}
logger.warn(
- `[createIndexesWithRetry] ${label} is waiting for another index build on the collection to finish (${Math.round(elapsedMs / 1000)}s elapsed, next attempt in ${pollMs}ms)`,
+ `[buildIndexWithRetry] ${label} is waiting for another index build on the collection to finish (${Math.round(elapsedMs / 1000)}s elapsed, next attempt in ${pollMs}ms)`,
);
await new Promise((resolve) => setTimeout(resolve, pollMs));
}
}
}
+/**
+ * Runs an index build with peer-build polling and transient-error retry, for a
+ * caller that holds a raw driver collection or a third-party builder rather
+ * than a Mongoose model. `build` is re-invoked until it settles, so it must be
+ * idempotent — which every `createIndex` of an existing spec is.
+ * Use this (or `createIndexesWithRetry`) instead of a raw `createIndex` on
+ * FerretDB or DocumentDB; the static guard in `methods/documentdb.spec.ts`
+ * rejects a raw call outside this module.
+ */
+export async function buildIndexWithRetry<T>(
+ build: () => Promise<T>,
+ label: string,
+ options: IndexBuildOptions = {},
+): Promise<T> {
+ return await retryWithBackoff(
+ () => buildWhenCollectionFree(build, label, options),
+ label,
+ options,
+ );
+}
+
/**
* Creates all indexes for a Mongoose model with deadlock retry.
* Use this instead of raw `model.createIndexes()` on FerretDB or DocumentDB.
@@ -160,10 +192,9 @@ export async function createIndexesWithRetry(
options: IndexBuildOptions = {},
): Promise<void> {
await settleAutomaticIndexBuild(model);
- const label = `createIndexes(${model.modelName})`;
- await retryWithBackoff(
- () => buildIndexesWhenCollectionFree(model, label, options),
- label,
+ await buildIndexWithRetry(
+ () => model.createIndexes(),
+ `createIndexes(${model.modelName})`,
options,
);
}