Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🍃 fix: Amazon DocumentDB Compatibility for Pipeline-Form Updates (#14495)
Danny Avila23d1ad473daa
diff --git a/.github/workflows/eslint-ci.yml b/.github/workflows/eslint-ci.ymlindex 115ad522dd..20c0aaf876 100644--- a/.github/workflows/eslint-ci.yml+++ b/.github/workflows/eslint-ci.yml@@ -61,8 +61,11 @@ jobs:fi# Run ESLint+ # --no-warn-ignored: changed files under config-ignored paths+ # (e.g. packages/data-schemas/misc/**) must not fail --max-warnings=0npx eslint --no-error-on-unmatched-pattern \--config eslint.config.mjs \+ --no-warn-ignored \--max-warnings=0 \-- "${CHANGED_FILES[@]}"diff --git a/packages/data-schemas/misc/documentdb/compat.documentdb.spec.ts b/packages/data-schemas/misc/documentdb/compat.documentdb.spec.tsnew file mode 100644index 0000000000..e31db558c7--- /dev/null+++ b/packages/data-schemas/misc/documentdb/compat.documentdb.spec.ts@@ -0,0 +1,234 @@+import mongoose from 'mongoose';+import { randomUUID } from 'crypto';+import type { ConnectOptions, Model } from 'mongoose';+import type { IConversationTag } from '~/schema/conversationTag';+import type * as t from '~/types';+import { decrementTagCounts } from '~/methods/conversationTag';+import { supportsTransactions } from '~/utils/transactions';+import { createUserMethods } from '~/methods/user';+import { createFileMethods } from '~/methods/file';+import { createModels } from '~/models';++/**+ * Amazon DocumentDB live-compatibility suite.+ *+ * Exercises the operations that have historically broken on DocumentDB+ * (aggregation-pipeline updates — issue #14488) against a REAL cluster, plus+ * informational capability probes whose results print as a matrix at the end.+ * There is no faithful local emulator of Amazon DocumentDB (the open-source+ * "DocumentDB Local" image is an unrelated PostgreSQL-based engine), so this+ * suite only runs when DOCUMENTDB_URI is set and skips otherwise.+ *+ * Run (from packages/data-schemas, against a DEDICATED database):+ * DOCUMENTDB_URI="mongodb://user:[email protected]:27017/librechat_compat?tls=true&retryWrites=false" \+ * DOCUMENTDB_TLS_CA_FILE="global-bundle.pem" \+ * npx jest --config misc/documentdb/jest.documentdb.config.mjs+ *+ * Through an SSH tunnel, additionally set+ * DOCUMENTDB_TLS_ALLOW_INVALID_HOSTNAMES=true+ * because the tunnel endpoint will not match the cluster certificate.+ *+ * Set DOCUMENTDB_EXPECT_PARTIAL_INDEXES=true when targeting DocumentDB 5.0++ * instance-based clusters to turn the partial-index probe into a hard assertion.+ */+const DOCUMENTDB_URI = process.env.DOCUMENTDB_URI ?? '';+const describeLive = DOCUMENTDB_URI ? describe : describe.skip;++const HOUR = 3_600_000;+const runId = randomUUID().slice(0, 8);+const capabilities: Record<string, string> = {};++function getDb() {+ const db = mongoose.connection.db;+ if (!db) {+ throw new Error('MongoDB database handle not available');+ }+ return db;+}++describeLive('Amazon DocumentDB live compatibility', () => {+ let User: Model<t.IUser>;+ let ConversationTag: Model<IConversationTag>;+ let userMethods: ReturnType<typeof createUserMethods>;+ let fileMethods: ReturnType<typeof createFileMethods>;++ const testEmail = (label: string) => `${label}-${runId}@compat.test`;++ beforeAll(async () => {+ const options: ConnectOptions = { autoIndex: false, autoCreate: false };+ if (process.env.DOCUMENTDB_TLS_CA_FILE) {+ options.tlsCAFile = process.env.DOCUMENTDB_TLS_CA_FILE;+ }+ if (process.env.DOCUMENTDB_TLS_ALLOW_INVALID_HOSTNAMES === 'true') {+ options.tlsAllowInvalidHostnames = true;+ }+ await mongoose.connect(DOCUMENTDB_URI, options);++ const models = createModels(mongoose);+ Object.assign(mongoose.models, models);+ User = mongoose.models.User;+ ConversationTag = mongoose.models.ConversationTag;+ userMethods = createUserMethods(mongoose);+ fileMethods = createFileMethods(mongoose);+ });++ afterAll(async () => {+ if (mongoose.connection.readyState === 1) {+ await User.deleteMany({ email: { $regex: runId } });+ await ConversationTag.deleteMany({ tag: { $regex: runId } });+ await mongoose.models.File.deleteMany({ filename: { $regex: runId } });++ const rows = Object.entries(capabilities).map(+ ([capability, verdict]) => ` ${capability.padEnd(36)} ${verdict}`,+ );+ console.log(`\nDocumentDB capability matrix (run ${runId}):\n${rows.join('\n')}\n`);+ await mongoose.disconnect();+ }+ });++ describe('pipeline-form updates (bug class behind #14488)', () => {+ it('records whether the engine accepts pipeline updates and $$NOW', async () => {+ const probe = getDb().collection(`pipeline_probe_${runId}`);+ await probe.insertOne({ probe: 1 });++ capabilities['pipeline-form updateOne'] = await probe+ .updateOne({ probe: 1 }, [{ $set: { probed: true } }])+ .then(() => 'supported')+ .catch((error: Error) => `rejected (${error.message})`);+ capabilities['$$NOW system variable'] = await probe+ .updateOne({ probe: 1 }, [{ $set: { probedAt: '$$NOW' } }])+ .then(() => 'supported')+ .catch((error: Error) => `rejected (${error.message})`);++ await probe.drop().catch(() => undefined);+ expect(capabilities['pipeline-form updateOne']).toBeDefined();+ });++ it('acceptTerms stamps once and preserves the first timestamp', async () => {+ const user = await User.create({+ name: 'DocDB Terms',+ email: testEmail('terms'),+ provider: 'local',+ });+ const userId = String(user._id);++ const first = await userMethods.acceptTerms(userId);+ expect(first?.termsAccepted).toBe(true);+ expect(first?.termsAcceptedAt).toBeInstanceOf(Date);++ const repeat = await userMethods.acceptTerms(userId);+ expect((repeat?.termsAcceptedAt as Date).getTime()).toBe(+ (first?.termsAcceptedAt as Date).getTime(),+ );+ });++ it('acceptTerms converges under concurrent requests', async () => {+ const user = await User.create({+ name: 'DocDB Concurrent',+ email: testEmail('concurrent'),+ provider: 'local',+ });+ const userId = String(user._id);++ const results = await Promise.all(+ Array.from({ length: 5 }, () => userMethods.acceptTerms(userId)),+ );++ expect(results.every((result) => result?.termsAccepted === true)).toBe(true);+ const stamped = new Set(results.map((result) => (result?.termsAcceptedAt as Date).getTime()));+ expect(stamped.size).toBe(1);+ });++ it('decrementTagCounts clamps at zero', async () => {+ const user = `docdb-user-${runId}`;+ const tag = `tag-${runId}`;+ await ConversationTag.create({ user, tag, position: 1, count: 1 });++ await decrementTagCounts(mongoose, user, [tag, tag, tag]);++ const stored = await ConversationTag.findOne({ user, tag }).lean();+ expect(stored?.count).toBe(0);+ });++ it('extendFilesTTL widens toward the window and clamps to the ceiling', async () => {+ const userId = new mongoose.Types.ObjectId();+ const fileId = randomUUID();+ await fileMethods.createFile({+ file_id: fileId,+ user: userId,+ filename: `${fileId}-${runId}.txt`,+ filepath: `/uploads/${fileId}.txt`,+ type: 'text/plain',+ bytes: 1,+ });+ await mongoose.models.File.updateOne(+ { file_id: fileId },+ { $set: { expiresAt: new Date(Date.now() + 60_000) } },+ { timestamps: false },+ );++ const hold = { renewMs: 24 * HOUR, maxLifetimeMs: 48 * HOUR };+ const widened = await fileMethods.extendFilesTTL([fileId], hold, { user: String(userId) });+ expect(widened).toBe(1);++ const stored = await mongoose.models.File.findOne({ file_id: fileId }).lean<{+ createdAt: Date;+ expiresAt?: Date;+ }>();+ expect(stored?.expiresAt).toBeDefined();+ expect(stored!.expiresAt!.getTime()).toBeGreaterThan(Date.now() + 23 * HOUR);+ expect(stored!.expiresAt!.getTime()).toBeLessThanOrEqual(+ stored!.createdAt.getTime() + hold.maxLifetimeMs,+ );+ });+ });++ describe('capability probes (informational)', () => {+ it('probes multi-document transaction support', async () => {+ const supported = await supportsTransactions(mongoose);+ capabilities['multi-document transactions'] = supported+ ? 'supported'+ : 'unsupported (runtime fallback engages)';+ expect(typeof supported).toBe('boolean');+ });++ it('probes partial unique index support (OAuth id uniqueness relies on it)', async () => {+ const probe = getDb().collection(`partial_index_probe_${runId}`);+ await probe.insertOne({ seeded: true });++ const outcome = await probe+ .createIndex(+ { googleId: 1, tenantId: 1 },+ { unique: true, partialFilterExpression: { googleId: { $exists: true } } },+ )+ .then(() => 'supported')+ .catch((error: Error) => `REJECTED (${error.message})`);+ capabilities['partial unique indexes'] = outcome;++ await probe.drop().catch(() => undefined);+ if (process.env.DOCUMENTDB_EXPECT_PARTIAL_INDEXES === 'true') {+ expect(outcome).toBe('supported');+ }+ });++ it('verifies TTL index support (session/token expiry relies on it)', async () => {+ const probe = getDb().collection(`ttl_probe_${runId}`);+ await probe.insertOne({ createdAt: new Date() });++ await expect(+ probe.createIndex({ createdAt: 1 }, { expireAfterSeconds: 60 }),+ ).resolves.toBeDefined();+ capabilities['TTL indexes'] = 'supported';++ await probe.drop().catch(() => undefined);+ });++ it('flags a connection string missing retryWrites=false', () => {+ const disabled = /retryWrites=false/i.test(DOCUMENTDB_URI);+ capabilities['retryWrites=false in URI'] = disabled+ ? 'present'+ : 'MISSING — DocumentDB rejects retryable writes';+ expect(typeof disabled).toBe('boolean');+ });+ });+});diff --git a/packages/data-schemas/misc/documentdb/documentdb-compat.md b/packages/data-schemas/misc/documentdb/documentdb-compat.mdnew file mode 100644index 0000000000..d3569d9687--- /dev/null+++ b/packages/data-schemas/misc/documentdb/documentdb-compat.md@@ -0,0 +1,195 @@+# Amazon DocumentDB Compatibility Assessment (issue #14488)++Adjudicated against official AWS documentation on 2026-07-28. Engine columns+throughout: DocumentDB **3.6 / 4.0 / 5.0 / 8.0 instance-based** and **elastic+clusters**. AWS's supported-APIs page states that unsupported operators are+*omitted* from its tables, so several verdicts below are implicit-by-omission+and flagged as such.++## Executive summary++- The login blocker was real: three aggregation-pipeline updates (one using+ `$$NOW`) existed in the codebase. DocumentDB documents no support for+ pipeline-form updates on any engine version. All three are now rewritten+ with plain update operators that work on every engine, including elastic.+- Partial-index creation fails on DocumentDB < 5.0 and elastic — and the+ failure was **provably silent** (no log, no crash, uniqueness quietly+ unenforced). Model creation now attaches an `index` listener so failed+ builds log loudly.+- Transactions already degrade gracefully via a runtime probe. GridFS is+ unreachable dead code. `retryWrites=false` is a deployment-docs item.+- Recommendation: **DocumentDB 5.0+ instance-based is a supportable target**;+ 4.0 runs but with degraded uniqueness enforcement (now logged); **elastic+ clusters should be documented as unsupported** (no unique indexes at all).++## Proven incompatibilities — fixed in this PR++### 1. `acceptTerms` pipeline update + `$$NOW` (P0 — blocked login)++`packages/data-schemas/src/methods/user.ts:303` used a pipeline-form+`findByIdAndUpdate` with `$ifNull`/`$$NOW` (introduced by PR #10810, matching+the reporter's regression window). When Terms gating is on, every login hits+this and DocumentDB rejects it.++AWS evidence: the [supported APIs page](https://docs.aws.amazon.com/documentdb/latest/developerguide/mongo-apis.html)+lists only classic update operators (no pipeline form anywhere); the `$set`/+`$unset` *stage* operators are marked unsupported for 3.6/4.0/5.0; `$$NOW` is+absent from the System variables table entirely (`$$CURRENT` and `$$REMOVE`+are explicitly "No"). Implicit-by-omission, but consistent with the reported+`Failed to parse update: field must be of BSON type object` class of error+(AWS documents no exact error string).++Fix: null-guarded first-acceptance claim (`termsAcceptedAt: null` matches both+the schema's explicit `null` default and missing legacy fields — a+`$exists: false` guard would never fire because of that default), with a+plain-`$set` fallback for repeat acceptance. First-acceptance timestamp+preservation, concurrency convergence, and the `IUser | null` contract are+covered by tests, including a raw-inserted legacy document without the field.++### 2. `decrementTagCounts` pipeline update (P1 — silent tag-count drift)++`packages/data-schemas/src/methods/conversationTag.ts:47` used a+`$max`/`$subtract`/`$ifNull` pipeline inside `bulkWrite`, wrapped in a+try/catch that only logs — on DocumentDB, conversation deletion succeeded+while tag counts silently drifted.++Fix: two mutually exclusive plain ops per tag in one ordered `bulkWrite` —+clamp-to-zero (`count` below the decrement amount, or null/missing) first,+then a guarded `$inc`. Clamp-at-zero, missing-count tolerance, and+variable-amount semantics all preserved; now covered by a new test block+(previously untested).++### 3. `extendFilesTTL` pipeline update (P1 — `/files/usage` TTL holds fail)++`packages/data-schemas/src/methods/file.ts:607` — **not in the reporter's+list; found by sweeping the codebase** (`rg` for pipeline-shaped update args+and `$$NOW`; these three sites were the only hits).++Fix: read the candidate files (one projected query), compute each file's+`min(now + renewMs, createdAt + maxLifetimeMs)` ceiling client-side, then+issue per-document guarded `$set`s (`expiresAt: { $exists: true, $lt: next }`)+through `tenantSafeBulkWrite`. Only-widens, per-file ceiling, and+cleared-TTL-stays-permanent semantics are preserved under concurrency by the+write guard. Cost: one extra read round trip on this path — unavoidable+without a schema change, because the ceiling is per-document.++## Proven, made loud — partial indexes (P1 on < 5.0 / elastic)++Four unique partial indexes exist:++- `packages/data-schemas/src/schema/user.ts:192` and `:199` — OAuth ids+ (`googleId`, `openidId`, …) with `partialFilterExpression: { $exists: true }`+- `packages/data-schemas/src/schema/file.ts:173` — `execute_code` files+ (`$eq`-shaped filter)+- `packages/data-schemas/src/schema/group.ts:56` — group source ids+ (`$exists: true`)++AWS: [partial-index.html](https://docs.aws.amazon.com/documentdb/latest/developerguide/partial-index.html)+— "The partial index feature is supported in Amazon DocumentDB 5.0+instance-based clusters"; the index-properties table marks Partial as+No/No/Yes/Yes/No across 3.6/4.0/5.0/8.0/elastic. The `$exists` and `$eq`+filter shapes used here are inside DocumentDB 5.0's supported operator list+(`$eq, $exists, $and, $gt/$gte/$lt/$lte`), so on 5.0+ these indexes build.++On 3.6/4.0/elastic the builds fail — and empirically (probe: unique index over+pre-seeded duplicates, mongoose 8, autoIndex) the failure is **completely+silent**: no unhandled rejection, no log, the index simply doesn't exist and+duplicate inserts succeed. Mongoose only surfaces build errors through a+`Model.on('index')` listener, which nothing attached. `createModels` now+attaches one that logs every failed build (`packages/data-schemas/src/models/index.ts`).+Operational consequence on < 5.0 remains: OAuth-account uniqueness is not+DB-enforced — documented, loud, but not fixable in application code.++## Proven compatible — no action needed++- **Transactions**: supported on 4.0+ instance-based ("Amazon DocumentDB …+ supports transactions in Amazon DocumentDB 4.0 and later" —+ [transactions.html](https://docs.aws.amazon.com/documentdb/latest/developerguide/transactions.html));+ unsupported on 3.6 and elastic. LibreChat already probes at runtime+ (`packages/data-schemas/src/utils/transactions.ts`, cached in+ `api/server/services/PermissionService.js`) and falls back to+ non-transactional writes — the same mode as standalone MongoDB without a+ replica set. DocumentDB's restrictions (1-minute execution limit, no cursors+ in transactions, no retryable commit/abort) don't intersect LibreChat's+ usage.+- **GridFS**: `packages/api/src/cache/keyvMongo.ts` only constructs a+ `GridFSBucket` when `useGridFS` is set — no caller ever sets it and the+ class isn't exported (the singleton uses a plain `logs` collection). Dead+ code. Moot regardless: AWS lists GridFS as supported on instance-based+ clusters (elastic: no).+- **TTL indexes**: supported everywhere including elastic. AWS warns deletion+ is best-effort ("Documents are not guaranteed to be deleted within any+ specific period") — acceptable, since LibreChat treats TTL as cleanup, not+ as a security boundary.+- **`$ifNull`**: supported on all versions (only its pipeline-update context+ was the problem).++## Deployment requirements (documentation, not code)++- **`retryWrites=false` is mandatory in `MONGO_URI`.** AWS: "Amazon DocumentDB+ does not currently support retryable writes"; the failure mode is+ `{"ok":0,"errmsg":"Unrecognized field: 'txnNumber'","code":9}`+ ([functional-differences.html](https://docs.aws.amazon.com/documentdb/latest/developerguide/functional-differences.html)).+ `api/db/connect.js` passes `MONGO_URI` through verbatim, so this belongs in+ the deployment docs (and the live harness flags a URI missing it).+- TLS with the AWS CA bundle; clusters are VPC-only (tunnel/bastion for+ external access).++## Document as unsupported — elastic clusters++[Elastic cluster limitations](https://docs.aws.amazon.com/documentdb/latest/developerguide/docdb-using-elastic-clusters.html):+no **unique indexes** (any), no partial indexes, no ACID transactions, no+GridFS, no change streams, `$expr` unsupported, and the cursor-methods table+even lists `sort()`/`skip()`/`limit()` as "No". The `email + tenantId` unique+index alone disqualifies elastic clusters. Recommend stating this explicitly+in the docs.++## Undetermined — honest gaps++- **The reporter's engine version and cluster type** — still unknown; it+ decides whether the partial-index caveat applies to them (5.0+: it doesn't).+ Worth asking directly on the issue.+- **DocumentDB 8.0 pipeline-update acceptance** — 8.0 added `$set`/`$unset`+ aggregation *stages*, but AWS never documents pipeline-form updates; the+ harness probe answers this live.+- **`collMod` is only "Partial"** on every version — avoid+ `Model.syncIndexes()` against DocumentDB (it may issue `collMod` beyond the+ documented `expireAfterSeconds`).+- **Read-side aggregations** (3 files: `methods/prompt.ts`,+ `methods/aclEntry.ts`, `methods/agentCategory.ts`) were not audited+ stage-by-stage; no exotic stages (`$facet`, `$setWindowFields`,+ `$unionWith`, `$graphLookup`) are used anywhere.+- **No faithful local emulator exists.** The `documentdb-local` Docker image+ is the PostgreSQL-based Linux Foundation project — AWS's own OSS blog+ confirms "a different engine than the one used in Amazon DocumentDB." Live+ regression testing must run against a real cluster.++## Regression strategy++1. **In-repo (CI today)**: the behavioral tests added in+ `user.methods.spec.ts`, `conversationTag.methods.spec.ts`, and+ `file.spec.ts` lock the pipeline-free implementations' semantics+ (first-acceptance preservation, clamp-at-zero, per-file ceiling).+2. **Live harness (this directory)**: `compat.documentdb.spec.ts` exercises+ the exact operations behind #14488 against a real cluster and prints a+ capability matrix (pipeline updates, `$$NOW`, transactions, partial unique+ indexes, TTL, `retryWrites`). Gated on `DOCUMENTDB_URI`; verified green+ against real MongoDB as a baseline. Suggested cadence: before releases and+ whenever update-operator code in `data-schemas` changes; optionally a+ scheduled GitHub Action on a runner with VPC access to a dev cluster.++## Support matrix and recommendation++| Capability (LibreChat dependency) | 3.6 | 4.0 | 5.0 | 8.0 | Elastic |+| --- | --- | --- | --- | --- | --- |+| Pipeline updates (**no longer used**) | ✗ | ✗ | ✗ | ? | ✗ |+| Plain update operators (all writes now) | ✓ | ✓ | ✓ | ✓ | ✓ |+| Unique indexes | ✓ | ✓ | ✓ | ✓ | ✗ |+| Partial unique indexes (OAuth ids) | ✗ | ✗ | ✓ | ✓ | ✗ |+| Transactions (runtime-probed) | ✗ | ✓ | ✓ | ✓ | ✗ |+| TTL indexes | ✓ | ✓ | ✓ | ✓ | ✓ |++**Recommendation**: support **DocumentDB 5.0+ instance-based** with+`retryWrites=false` documented as required. 4.0 functions with+partial-unique-index loss (now logged loudly at startup) — "works, with a+documented caveat". Elastic clusters: unsupported, full stop.diff --git a/packages/data-schemas/misc/documentdb/jest.documentdb.config.mjs b/packages/data-schemas/misc/documentdb/jest.documentdb.config.mjsnew file mode 100644index 0000000000..3a149c0863--- /dev/null+++ b/packages/data-schemas/misc/documentdb/jest.documentdb.config.mjs@@ -0,0 +1,20 @@+/**+ * Jest config for Amazon DocumentDB live-compatibility tests.+ * These tests require network access to a real Amazon DocumentDB cluster+ * (VPC-only — run from a bastion/tunnel) and are NOT run in CI by default.+ *+ * Usage:+ * DOCUMENTDB_URI="mongodb://user:[email protected]:27017/librechat_compat?tls=true&retryWrites=false" \+ * DOCUMENTDB_TLS_CA_FILE="global-bundle.pem" \+ * npx jest --config misc/documentdb/jest.documentdb.config.mjs+ */+export default {+ rootDir: '../..',+ testMatch: ['<rootDir>/misc/documentdb/**/*.documentdb.spec.ts'],+ moduleNameMapper: {+ '^@src/(.*)$': '<rootDir>/src/$1',+ '^~/(.*)$': '<rootDir>/src/$1',+ },+ restoreMocks: true,+ testTimeout: 120000,+};diff --git a/packages/data-schemas/misc/documentdb/tsconfig.json b/packages/data-schemas/misc/documentdb/tsconfig.jsonnew file mode 100644index 0000000000..a64965da40--- /dev/null+++ b/packages/data-schemas/misc/documentdb/tsconfig.json@@ -0,0 +1,13 @@+{+ "extends": "../../tsconfig.json",+ "compilerOptions": {+ "noEmit": true,+ "target": "ES2020",+ "lib": ["ES2020"],+ "paths": {+ "~/*": ["../../src/*"]+ }+ },+ "include": ["./**/*.ts"],+ "exclude": ["node_modules"]+}diff --git a/packages/data-schemas/src/methods/conversationTag.methods.spec.ts b/packages/data-schemas/src/methods/conversationTag.methods.spec.tsindex 0b4c6268d6..cbc1990973 100644--- a/packages/data-schemas/src/methods/conversationTag.methods.spec.ts+++ b/packages/data-schemas/src/methods/conversationTag.methods.spec.ts@@ -1,9 +1,9 @@import mongoose from 'mongoose';import { MongoMemoryServer } from 'mongodb-memory-server';-import { createConversationTagMethods } from './conversationTag';-import { createModels } from '~/models';-import type { IConversationTag } from '~/schema/conversationTag';import type { IConversation } from '..';+import type { IConversationTag } from '~/schema/conversationTag';+import { createConversationTagMethods, decrementTagCounts } from './conversationTag';+import { createModels } from '~/models';jest.mock('~/config/winston', () => ({error: jest.fn(),@@ -137,3 +137,84 @@ describe('ConversationTag model - $pullAll operations', () => {});});});++describe('decrementTagCounts', () => {+ const userId = new mongoose.Types.ObjectId().toString();++ const readCount = async (tag: string, user: string = userId) =>+ (await ConversationTag.findOne({ user, tag }).lean())?.count;++ it('decrements once per tag occurrence', async () => {+ await ConversationTag.create({ tag: 'work', user: userId, position: 1, count: 5 });++ await decrementTagCounts(mongoose, userId, ['work', 'work']);++ expect(await readCount('work')).toBe(3);+ });++ it('decrements an exact count down to zero', async () => {+ await ConversationTag.create({ tag: 'work', user: userId, position: 1, count: 2 });++ await decrementTagCounts(mongoose, userId, ['work', 'work']);++ expect(await readCount('work')).toBe(0);+ });++ it('clamps at zero when the decrement exceeds the current count', async () => {+ await ConversationTag.create({ tag: 'work', user: userId, position: 1, count: 1 });++ await decrementTagCounts(mongoose, userId, ['work', 'work', 'work']);++ expect(await readCount('work')).toBe(0);+ });++ it('leaves a zero count at zero', async () => {+ await ConversationTag.create({ tag: 'empty', user: userId, position: 1, count: 0 });++ await decrementTagCounts(mongoose, userId, ['empty']);++ expect(await readCount('empty')).toBe(0);+ });++ it('clamps pre-existing negative drift to zero', async () => {+ await ConversationTag.create({ tag: 'drift', user: userId, position: 1, count: -3 });++ await decrementTagCounts(mongoose, userId, ['drift']);++ expect(await readCount('drift')).toBe(0);+ });++ it('treats a missing count as zero', async () => {+ await ConversationTag.collection.insertOne({ tag: 'legacy', user: userId, position: 1 });++ await decrementTagCounts(mongoose, userId, ['legacy']);++ expect(await readCount('legacy')).toBe(0);+ });++ it('converges to zero under concurrent decrements exceeding the count', async () => {+ await ConversationTag.create({ tag: 'race', user: userId, position: 1, count: 3 });++ await Promise.all([+ decrementTagCounts(mongoose, userId, ['race', 'race']),+ decrementTagCounts(mongoose, userId, ['race', 'race']),+ ]);++ expect(await readCount('race')).toBe(0);+ });++ it("leaves other users' tags untouched", async () => {+ const otherUserId = new mongoose.Types.ObjectId().toString();+ await ConversationTag.create({ tag: 'work', user: userId, position: 1, count: 4 });+ await ConversationTag.create({ tag: 'work', user: otherUserId, position: 1, count: 4 });++ await decrementTagCounts(mongoose, userId, ['work']);++ expect(await readCount('work')).toBe(3);+ expect(await readCount('work', otherUserId)).toBe(4);+ });++ it('ignores empty and unknown tags without throwing', async () => {+ await expect(decrementTagCounts(mongoose, userId, ['', 'nonexistent'])).resolves.not.toThrow();+ });+});diff --git a/packages/data-schemas/src/methods/conversationTag.ts b/packages/data-schemas/src/methods/conversationTag.tsindex be65f6b446..cc24070b9a 100644--- a/packages/data-schemas/src/methods/conversationTag.ts+++ b/packages/data-schemas/src/methods/conversationTag.ts@@ -17,6 +17,16 @@ interface IConversationTag {* single decrement, so callers must dedupe tags per conversation before flattening* to avoid double-decrementing a conversation's duplicate tag entries. Counts are* clamped at zero to tolerate any pre-existing drift.+ *+ * Each tag emits three ops in one ordered bulkWrite instead of a+ * `$max`/`$subtract` aggregation-pipeline update (which Amazon DocumentDB+ * rejects): normalize a null/missing count to zero, apply the `$inc`, then+ * clamp a negative result back to zero. The clamp keys on `count < 0` rather+ * than `count < amount` so it composes with concurrent decrements of the same+ * tag: increments commute and every interleaved call ends with its own clamp,+ * so the count still converges on `max(0, ...)` exactly as the serialized+ * pipeline did. The only trade-off is a transiently negative count between an+ * op pair, which readers already tolerate.*/export async function decrementTagCounts(mongoose: typeof import('mongoose'),@@ -41,18 +51,26 @@ export async function decrementTagCounts(try {const ConversationTag = mongoose.models.ConversationTag as Model<IConversationTag>;- const bulkOps = [...decrementByTag.entries()].map(([tag, amount]) => ({- updateOne: {- filter: { user, tag },- update: [- {- $set: {- count: { $max: [0, { $subtract: [{ $ifNull: ['$count', 0] }, amount] }] },- },- },- ],+ const bulkOps = [...decrementByTag.entries()].flatMap(([tag, amount]) => [+ {+ updateOne: {+ filter: { user, tag, count: null },+ update: { $set: { count: 0 } },+ },+ },+ {+ updateOne: {+ filter: { user, tag },+ update: { $inc: { count: -amount } },+ },+ },+ {+ updateOne: {+ filter: { user, tag, count: { $lt: 0 } },+ update: { $set: { count: 0 } },+ },},- }));+ ]);await tenantSafeBulkWrite(ConversationTag, bulkOps);} catch (error) {diff --git a/packages/data-schemas/src/methods/file.spec.ts b/packages/data-schemas/src/methods/file.spec.tsindex 4aa424301b..54eece631b 100644--- a/packages/data-schemas/src/methods/file.spec.ts+++ b/packages/data-schemas/src/methods/file.spec.ts@@ -1143,9 +1143,17 @@ describe('File Methods', () => {});await mongoose.models.File.updateOne({ file_id: fileId },- { $set: { expiresAt, ...(createdAt ? { createdAt } : {}) } },+ { $set: { expiresAt } },{ timestamps: false },);+ /** Mongoose strips the immutable `createdAt` from updates, so backdating+ * must go through the raw driver to actually land. */+ if (createdAt) {+ await mongoose.models.File.collection.updateOne(+ { file_id: fileId },+ { $set: { createdAt } },+ );+ }return fileId;};@@ -1214,6 +1222,28 @@ describe('File Methods', () => {expect(file.expiresAt!.getTime()).toBeGreaterThan(Date.now() + 50 * 60_000);});+ it("applies each file's own ceiling within a single batch", async () => {+ const userId = new mongoose.Types.ObjectId();+ const soon = new Date(Date.now() + 60_000);+ const freshId = await seedTempFile(userId, soon);+ const agedId = await seedTempFile(userId, soon, new Date(Date.now() - 47 * HOUR));+ const releasedId = await seedTempFile(userId, soon);+ await fileMethods.updateFileUsage({ file_id: releasedId, user: String(userId) });++ const count = await fileMethods.extendFilesTTL([freshId, agedId, releasedId], HOLD, {+ user: String(userId),+ });++ expect(count).toBe(2);+ const fresh = await readFile(freshId);+ expect(fresh.expiresAt!.getTime()).toBeGreaterThan(Date.now() + 23 * HOUR);+ const aged = await readFile(agedId);+ expect(aged.expiresAt!.getTime()).toBe(aged.createdAt.getTime() + HOLD.maxLifetimeMs);+ expect(aged.expiresAt!.getTime()).toBeLessThan(Date.now() + 2 * HOUR);+ const released = await readFile(releasedId);+ expect(released.expiresAt).toBeUndefined();+ });+it('does not resurrect a TTL on an already-released file', async () => {const userId = new mongoose.Types.ObjectId();const fileId = await seedTempFile(userId, new Date(Date.now() + 60_000));diff --git a/packages/data-schemas/src/methods/file.ts b/packages/data-schemas/src/methods/file.tsindex ac428bdce0..cb544862c1 100644--- a/packages/data-schemas/src/methods/file.ts+++ b/packages/data-schemas/src/methods/file.ts@@ -557,20 +557,23 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {** A renewable hold, not a release: unlike `updateFileUsage` this never* unsets `expiresAt`, so a file that is held but never actually sent is- * still reaped once the hold lapses. Four properties hold by- * construction, which is what makes the write safe to drive from a- * client-supplied id list:- * - `$min` against `createdAt + maxLifetimeMs` caps every renewal against- * an immutable anchor, so repeated calls converge on a fixed ceiling+ * still reaped once the hold lapses. Candidates are read first, then each+ * doc gets a guarded write — no aggregation-pipeline update, which Amazon+ * DocumentDB rejects. Four properties hold by construction, which is what+ * makes the write safe to drive from a client-supplied id list:+ * - capping the renewal at `createdAt + maxLifetimeMs` anchors it to an+ * immutable ceiling, so repeated calls converge on a fixed deadline* instead of walking a file's lifetime forward a window at a time;* - renewing from `now` up to that ceiling lets a queue that is still* draining keep its attachments alive across successive runs, while an* abandoned queue lapses a single `renewMs` after its last touch rather* than surviving to the ceiling;- * - `$max` against the current value means a hold only ever widens;- * - `expiresAt: { $exists: true }` means a file whose TTL was already- * cleared by a real send stays permanent. Re-adding `expiresAt` there- * would schedule a live file for deletion.+ * - the `expiresAt: { $lt: next }` write guard means a hold only ever+ * widens, even against renewals landing between the read and the write;+ * - `expiresAt: { $exists: true }` in the read filter and the write guard+ * means a file whose TTL was already cleared by a real send stays+ * permanent. Re-adding `expiresAt` there would schedule a live file for+ * deletion.** `createdAt` is required rather than defaulted: without the anchor there* is no ceiling to enforce, so such a file is skipped instead of held.@@ -603,23 +606,34 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {},{ userId: owner.user, tenantId: owner.tenantId },);- const renewUntil = new Date(Date.now() + renewMs);- const result = await File.updateMany(- filter,- [+ const renewUntil = Date.now() + renewMs;+ const candidates = await File.find(filter)+ .select({ _id: 1, expiresAt: 1, createdAt: 1 })+ .lean<Pick<IMongoFile, '_id' | 'expiresAt' | 'createdAt'>[]>();+ const holdOps = candidates.flatMap((file) => {+ if (!file.createdAt || !file.expiresAt) {+ return [];+ }+ const next = new Date(Math.min(renewUntil, file.createdAt.getTime() + maxLifetimeMs));+ if (file.expiresAt.getTime() >= next.getTime()) {+ return [];+ }+ return [{- $set: {- expiresAt: {- $max: ['$expiresAt', { $min: [renewUntil, { $add: ['$createdAt', maxLifetimeMs] }] }],- },+ updateOne: {+ filter: { _id: file._id, expiresAt: { $exists: true, $lt: next } },+ update: { $set: { expiresAt: next } },},},- ],- /** `timestamps: false`: a hold is TTL bookkeeping, not a content write.- * Bumping `updatedAt` would also make every re-touch count as a- * modification, hiding whether the deadline actually moved. */- { timestamps: false },- );+ ];+ });+ if (holdOps.length === 0) {+ return 0;+ }+ /** `timestamps: false`: a hold is TTL bookkeeping, not a content write.+ * Bumping `updatedAt` would also make every re-touch count as a+ * modification, hiding whether the deadline actually moved. */+ const result = await tenantSafeBulkWrite(File, holdOps, { timestamps: false });return result.modifiedCount ?? 0;}diff --git a/packages/data-schemas/src/methods/user.methods.spec.ts b/packages/data-schemas/src/methods/user.methods.spec.tsindex 9f15d2f9a4..f320c49e4a 100644--- a/packages/data-schemas/src/methods/user.methods.spec.ts+++ b/packages/data-schemas/src/methods/user.methods.spec.ts@@ -498,6 +498,65 @@ describe('User Methods - Database Tests', () => {expect(result).toBeNull();});+ test('stamps a fresh termsAcceptedAt when accepting after a terms reset', async () => {+ const user = await User.create({+ name: 'Reset User',+ email: '[email protected]',+ provider: 'local',+ });+ const userId = user._id?.toString() ?? '';+ const first = await methods.acceptTerms(userId);++ await User.updateOne(+ { _id: userId },+ { $set: { termsAccepted: false, termsAcceptedAt: null } },+ );+ const reaccepted = await methods.acceptTerms(userId);++ expect(reaccepted?.termsAccepted).toBe(true);+ expect(reaccepted?.termsAcceptedAt).toBeInstanceOf(Date);+ expect((reaccepted?.termsAcceptedAt as Date).getTime()).toBeGreaterThanOrEqual(+ (first?.termsAcceptedAt as Date).getTime(),+ );+ });++ test('stamps termsAcceptedAt for a legacy document missing the field entirely', async () => {+ const legacyId = new mongoose.Types.ObjectId();+ await User.collection.insertOne({+ _id: legacyId,+ name: 'Pre-Terms User',+ email: '[email protected]',+ provider: 'local',+ });++ const updated = await methods.acceptTerms(legacyId.toString());++ expect(updated?.termsAccepted).toBe(true);+ expect(updated?.termsAcceptedAt).toBeInstanceOf(Date);+ });++ test('converges on a single termsAcceptedAt under concurrent acceptance', async () => {+ const user = await User.create({+ name: 'Concurrent User',+ email: '[email protected]',+ provider: 'local',+ });+ const userId = user._id?.toString() ?? '';++ const results = await Promise.all(+ Array.from({ length: 5 }, () => methods.acceptTerms(userId)),+ );++ expect(results.every((result) => result?.termsAccepted === true)).toBe(true);+ const stampedTimes = new Set(+ results.map((result) => (result?.termsAcceptedAt as Date).getTime()),+ );+ expect(stampedTimes.size).toBe(1);++ const repeat = await methods.acceptTerms(userId);+ expect((repeat?.termsAcceptedAt as Date).getTime()).toBe([...stampedTimes][0]);+ });+test('should invalidate cached auth user documents on acceptance', async () => {enableAuthUserDocCache();const user = await User.create({diff --git a/packages/data-schemas/src/methods/user.ts b/packages/data-schemas/src/methods/user.tsindex d044bf40d4..35b5be6cae 100644--- a/packages/data-schemas/src/methods/user.ts+++ b/packages/data-schemas/src/methods/user.ts@@ -296,28 +296,45 @@ export function createUserMethods(/*** Atomically records terms acceptance for a user.- * Sets termsAccepted and, only when no timestamp is already stored, stamps- * termsAcceptedAt with the server time so the first acceptance within a terms- * cycle is preserved across concurrent or repeated requests.+ * A null-guarded claim stamps termsAcceptedAt only when no timestamp is+ * already stored (explicit null from the schema default, a missing legacy+ * field, or a terms reset), so the first acceptance within a terms cycle is+ * preserved across concurrent or repeated requests. The repeat-acceptance+ * fallback is guarded by the exact complement (a non-null timestamp) so it+ * can never resurrect termsAccepted into a cycle that config/reset-terms.js+ * started between the two updates; when both guards miss because a reset+ * raced in, the claim retries and records a fresh stamped acceptance. Plain+ * updates are used instead of an aggregation pipeline with $$NOW, which+ * Amazon DocumentDB rejects.*/async function acceptTerms(userId: string): Promise<IUser | null> {const User = mongoose.models.User;- const updated = await User.findByIdAndUpdate(- userId,- [- {- $set: {- termsAccepted: true,- termsAcceptedAt: { $ifNull: ['$termsAcceptedAt', '$$NOW'] },- },- },- ],- { new: true, runValidators: true },- ).lean<IUser>();- if (updated) {- await invalidateAuthUserDocCache(userId);+ const maxAttempts = 3;+ for (let attempt = 0; attempt < maxAttempts; attempt++) {+ const firstAcceptance = await User.findOneAndUpdate(+ { _id: userId, termsAcceptedAt: null },+ { $set: { termsAccepted: true, termsAcceptedAt: new Date() } },+ { new: true, runValidators: true },+ ).lean<IUser>();+ if (firstAcceptance) {+ await invalidateAuthUserDocCache(userId);+ return firstAcceptance;+ }+ const reacceptance = await User.findOneAndUpdate(+ { _id: userId, termsAcceptedAt: { $ne: null } },+ { $set: { termsAccepted: true } },+ { new: true, runValidators: true },+ ).lean<IUser>();+ if (reacceptance) {+ await invalidateAuthUserDocCache(userId);+ return reacceptance;+ }+ const exists = await User.exists({ _id: userId });+ if (!exists) {+ return null;+ }}- return updated;+ return null;}/**diff --git a/packages/data-schemas/src/models/index.ts b/packages/data-schemas/src/models/index.tsindex 440106f2e0..ed1e56d9a5 100644--- a/packages/data-schemas/src/models/index.ts+++ b/packages/data-schemas/src/models/index.ts@@ -35,6 +35,7 @@ import { createUserModel } from './user';import { createRoleModel } from './role';import { createFileModel } from './file';import { createKeyModel } from './key';+import logger from '~/config/winston';/*** Creates all database models for all collections@@ -78,7 +79,7 @@ export function createModels(mongoose: typeof import('mongoose')): {Group: ReturnType<typeof createGroupModel>;Config: ReturnType<typeof createConfigModel>;} {- return {+ const models = {User: createUserModel(mongoose),Token: createTokenModel(mongoose),Session: createSessionModel(mongoose),@@ -117,4 +118,19 @@ export function createModels(mongoose: typeof import('mongoose')): {Group: createGroupModel(mongoose),Config: createConfigModel(mongoose),};+ /**+ * Background index builds fail silently unless an 'index' listener is+ * attached (e.g. Amazon DocumentDB <5.0 rejecting partialFilterExpression),+ * leaving unique constraints unenforced with no trace in the logs.+ */+ for (const model of Object.values(models)) {+ if (model.listenerCount('index') === 0) {+ model.on('index', (error?: Error) => {+ if (error) {+ logger.error(`Index build failed for "${model.modelName}": ${error.message}`);+ }+ });+ }+ }+ return models;}