Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🛂 feat: Audit Resource Sharing Changes and Fail Closed on Reversible Audit Writes (#17053)

Marco Beretta76192bdfd0ce
diff --git a/.env.example b/.env.example
index 96c2c2cb38..0c57c87bff 100644
--- a/.env.example
+++ b/.env.example
@@ -177,6 +177,17 @@ TRUST_PROXY=1
# instead of exiting the process. Not recommended for production unless necessary.
# CONTINUE_ON_UNCAUGHT_EXCEPTION=false
+#===============#
+# Audit Log #
+#===============#
+
+# System grants and resource sharing are audited before they take effect: when a grant or a
+# widening permission change cannot be audited, the request fails and nothing changes.
+# Revocations are never blocked by an audit outage; their entry is retried after the change.
+# Set to false to apply every change anyway and only log audit failures.
+# Default: true
+# AUDIT_LOG_FAIL_CLOSED=true
+
#===============#
# JSON Logging #
#===============#
diff --git a/api/server/controllers/PermissionsController.js b/api/server/controllers/PermissionsController.js
index 5188729a06..37f1f1ede6 100644
--- a/api/server/controllers/PermissionsController.js
+++ b/api/server/controllers/PermissionsController.js
@@ -9,7 +9,8 @@ const {
enrichRemoteAgentPrincipals,
createPrincipalSearch,
backfillRemoteAgentPermissions,
- auditInsightsPermissionChanges,
+ runAuditedPermissionUpdate,
+ permissionErrorCode,
getInsightsPrincipalState,
maskAgentInsightsBit,
sanitizeInsightsPermissionPrincipals,
@@ -17,7 +18,6 @@ const {
} = require('@librechat/api');
const {
bulkUpdateResourcePermissions,
- restoreInsightsPermissionChanges,
ensureGroupPrincipalExists,
getResourcePermissionsMap,
findAccessibleResources,
@@ -170,31 +170,24 @@ const updateResourcePermissions = async (req, res) => {
});
}
- const results = await bulkUpdateResourcePermissions({
- resourceType,
- resourceId,
- maxWriteAttempts: req.config?.config?.permissions?.maxWriteAttempts,
- updatedPrincipals: validatedPrincipals,
- revokedPrincipals,
- grantedBy: userId,
- });
-
- await auditInsightsPermissionChanges({
+ const results = await runAuditedPermissionUpdate({
req,
+ resourceType,
resourceId,
- changes: results.insightsChanges ?? [],
- failClosed: process.env.AUDIT_LOG_FAIL_CLOSED === 'true',
- deps: {
- getAgent: db.getAgent,
- recordAuditEntry: db.recordAuditEntry,
- restoreInsightsPermissionChanges: (changes) =>
- restoreInsightsPermissionChanges({
- resourceType: ResourceType.AGENT,
- resourceId,
- changes,
- }),
- logger,
- },
+ deps: { recordAuditEntry: db.recordAuditEntry, getAgent: db.getAgent, logger },
+ update: (beforeWrite, afterWrite) =>
+ bulkUpdateResourcePermissions({
+ resourceType,
+ resourceId,
+ maxWriteAttempts: req.config?.config?.permissions?.maxWriteAttempts,
+ leaseTtlMs: req.config?.config?.permissions?.leaseTtlMs,
+ leaseWaitMs: req.config?.config?.permissions?.leaseWaitMs,
+ updatedPrincipals: validatedPrincipals,
+ revokedPrincipals,
+ grantedBy: userId,
+ beforeWrite,
+ afterWrite,
+ }),
});
if (resourceType === ResourceType.CODE_ENVIRONMENT) {
@@ -238,6 +231,7 @@ const updateResourcePermissions = async (req, res) => {
res.status(error.statusCode ?? 400).json({
error: 'Failed to update permissions',
details: error.message,
+ ...permissionErrorCode(error),
});
}
};
diff --git a/api/server/controllers/__tests__/PermissionsController.spec.js b/api/server/controllers/__tests__/PermissionsController.spec.js
index 17248e61b6..65bc7989e9 100644
--- a/api/server/controllers/__tests__/PermissionsController.spec.js
+++ b/api/server/controllers/__tests__/PermissionsController.spec.js
@@ -33,11 +33,17 @@ jest.mock('~/server/services/Config', () => ({
}));
const mockBulkUpdateResourcePermissions = jest.fn();
-const mockRestoreInsightsPermissionChanges = jest.fn();
jest.mock('~/server/services/PermissionService', () => ({
- bulkUpdateResourcePermissions: (...args) => mockBulkUpdateResourcePermissions(...args),
- restoreInsightsPermissionChanges: (...args) => mockRestoreInsightsPermissionChanges(...args),
+ /** The service audits through `beforeWrite` before writing; the mock result stands in for the computed plan. */
+ bulkUpdateResourcePermissions: async (params) => {
+ const results = await mockBulkUpdateResourcePermissions(params);
+ await params.beforeWrite?.({
+ aclChanges: results?.aclChanges ?? [],
+ insightsChanges: results?.insightsChanges ?? [],
+ });
+ return results;
+ },
ensureGroupPrincipalExists: jest.fn(),
getEffectivePermissions: jest.fn(),
ensurePrincipalExists: jest.fn(),
@@ -249,7 +255,6 @@ describe('PermissionsController', () => {
mockRemoveAgentFromUserFavorites.mockResolvedValue(undefined);
db.getAgent.mockResolvedValue({ _id: agentObjectId, id: 'agent-a', name: 'Agent A' });
mockRecordAuditEntry.mockResolvedValue({});
- mockRestoreInsightsPermissionChanges.mockResolvedValue(undefined);
});
it('rejects Insights permission changes from non-admin users', async () => {
@@ -339,20 +344,25 @@ describe('PermissionsController', () => {
});
const res = createMockRes();
- req.config = { config: { permissions: { maxWriteAttempts: 7 } } };
+ req.config = {
+ config: { permissions: { maxWriteAttempts: 7, leaseTtlMs: 45000, leaseWaitMs: 1500 } },
+ };
await updateResourcePermissions(req, res);
expect(mockBulkUpdateResourcePermissions).toHaveBeenCalledWith(
- expect.objectContaining({ maxWriteAttempts: 7 }),
+ expect.objectContaining({ maxWriteAttempts: 7, leaseTtlMs: 45000, leaseWaitMs: 1500 }),
);
expect(mockRecordAuditEntry).toHaveBeenCalledWith(
expect.objectContaining({
action: 'permission.insights_assigned',
tenantId: 'tenant-a',
target: expect.objectContaining({ id: 'agent-a' }),
- metadata: { principalType: PrincipalType.ROLE, principalId: 'USER' },
+ metadata: expect.objectContaining({
+ principalType: PrincipalType.ROLE,
+ principalId: 'USER',
+ }),
}),
- { failClosed: false },
+ { failClosed: true },
);
expect(res.status).toHaveBeenCalledWith(200);
expect(res.json.mock.calls[0][0].results.principals[0]).toEqual(
@@ -360,7 +370,7 @@ describe('PermissionsController', () => {
);
});
- it('restores unaudited Insights transitions when fail-closed auditing fails', async () => {
+ it('fails the request before writing when fail-closed Insights auditing fails', async () => {
process.env.AUDIT_LOG_FAIL_CLOSED = 'true';
const changes = [{ action: 'assigned', principal: { type: PrincipalType.ROLE, id: 'USER' } }];
mockBulkUpdateResourcePermissions.mockResolvedValue({
@@ -390,15 +400,10 @@ describe('PermissionsController', () => {
await updateResourcePermissions(req, res);
- expect(mockRestoreInsightsPermissionChanges).toHaveBeenCalledWith({
- resourceType: ResourceType.AGENT,
- resourceId: agentObjectId,
- changes,
- });
expect(res.status).toHaveBeenCalledWith(500);
});
- it('keeps a successful permission response when audit target lookup fails open', async () => {
+ it('audits the Insights change with the agent id when the target lookup fails', async () => {
mockBulkUpdateResourcePermissions.mockResolvedValue({
granted: [],
updated: [],
@@ -417,7 +422,133 @@ describe('PermissionsController', () => {
await updateResourcePermissions(req, res);
- expect(mockRestoreInsightsPermissionChanges).not.toHaveBeenCalled();
+ expect(mockRecordAuditEntry).toHaveBeenCalledWith(
+ expect.objectContaining({
+ action: 'permission.insights_assigned',
+ target: expect.objectContaining({ id: agentObjectId }),
+ }),
+ { failClosed: true },
+ );
+ expect(res.status).toHaveBeenCalledWith(200);
+ });
+
+ it('audits ACL grants, revokes and public changes without names or emails', async () => {
+ const aclChanges = [
+ {
+ action: 'assigned',
+ previous: null,
+ written: { permBits: 3, roleId: 'role-1' },
+ principal: { type: PrincipalType.USER, id: revokedUserId, name: 'Jane Doe' },
+ },
+ {
+ action: 'removed',
+ previous: { permBits: 1, roleId: 'role-2' },
+ written: null,
+ principal: { type: PrincipalType.GROUP, id: 'group-1', name: 'Team' },
+ },
+ {
+ action: 'assigned',
+ previous: null,
+ written: { permBits: 1, roleId: 'role-2' },
+ principal: { type: PrincipalType.PUBLIC, id: null },
+ },
+ ];
+ mockBulkUpdateResourcePermissions.mockResolvedValue({
+ granted: [],
+ updated: [],
+ revoked: [],
+ aclChanges,
+ insightsChanges: [],
+ errors: [],
+ });
+ const req = createMockReq({
+ params: { resourceType: ResourceType.PROMPTGROUP, resourceId: agentObjectId },
+ user: { id: 'user-id', name: 'Owner', role: SystemRoles.USER, tenantId: 'tenant-a' },
+ });
+ const res = createMockRes();
+
+ await updateResourcePermissions(req, res);
+
+ const entries = mockRecordAuditEntry.mock.calls.map(([input]) => input);
+ expect(entries.map((entry) => entry.action)).toEqual([
+ 'acl.granted',
+ 'acl.public_changed',
+ 'acl.revoked',
+ ]);
+ expect(entries[0]).toEqual(
+ expect.objectContaining({
+ target: { type: ResourceType.PROMPTGROUP, id: agentObjectId },
+ metadata: {
+ principalType: PrincipalType.USER,
+ principalId: revokedUserId,
+ permBits: 3,
+ previousPermBits: 0,
+ roleId: 'role-1',
+ previousRoleId: '',
+ resourceId: agentObjectId,
+ },
+ }),
+ );
+ expect(JSON.stringify(entries)).not.toMatch(/Jane Doe|Team/);
+ expect(res.status).toHaveBeenCalledWith(200);
+ });
+
+ it('fails the request before writing when the ACL audit write fails closed', async () => {
+ const aclChanges = [
+ {
+ action: 'assigned',
+ previous: null,
+ written: { permBits: 3, roleId: 'role-1' },
+ principal: { type: PrincipalType.USER, id: revokedUserId },
+ },
+ ];
+ mockBulkUpdateResourcePermissions.mockResolvedValue({
+ granted: [],
+ updated: [],
+ revoked: [],
+ aclChanges,
+ insightsChanges: [],
+ errors: [],
+ });
+ mockRecordAuditEntry.mockRejectedValue(new Error('audit unavailable'));
+ const req = createMockReq({
+ params: { resourceType: ResourceType.PROMPTGROUP, resourceId: agentObjectId },
+ user: { id: 'user-id', role: SystemRoles.USER },
+ });
+ const res = createMockRes();
+
+ await updateResourcePermissions(req, res);
+
+ expect(res.status).toHaveBeenCalledWith(500);
+ });
+
+ it('applies the permissions and logs when the audit write fails with the opt-out', async () => {
+ process.env.AUDIT_LOG_FAIL_CLOSED = 'false';
+ mockBulkUpdateResourcePermissions.mockResolvedValue({
+ granted: [],
+ updated: [],
+ revoked: [],
+ aclChanges: [
+ {
+ action: 'removed',
+ previous: { permBits: 1, roleId: 'role-2' },
+ written: null,
+ principal: { type: PrincipalType.USER, id: revokedUserId },
+ },
+ ],
+ insightsChanges: [],
+ errors: [],
+ });
+ mockRecordAuditEntry.mockRejectedValue(new Error('audit unavailable'));
+ const req = createMockReq({
+ params: { resourceType: ResourceType.PROMPTGROUP, resourceId: agentObjectId },
+ user: { id: 'user-id', role: SystemRoles.USER },
+ });
+ const res = createMockRes();
+
+ await updateResourcePermissions(req, res);
+
+ expect(mockRecordAuditEntry).toHaveBeenCalledWith(expect.anything(), { failClosed: false });
expect(res.status).toHaveBeenCalledWith(200);
});
diff --git a/api/server/routes/accessPermissions.js b/api/server/routes/accessPermissions.js
index 96aa8c8a3e..4429f1e4b3 100644
--- a/api/server/routes/accessPermissions.js
+++ b/api/server/routes/accessPermissions.js
@@ -19,7 +19,13 @@ const {
checkShareAccess,
checkSharePublicAccess,
} = require('~/server/middleware/checkSharePublicAccess');
-const { requireJwtAuth, checkBan, uaParser, canAccessResource } = require('~/server/middleware');
+const {
+ requireJwtAuth,
+ checkBan,
+ uaParser,
+ canAccessResource,
+ configMiddleware,
+} = require('~/server/middleware');
const { checkPeoplePickerAccess } = require('~/server/middleware/checkPeoplePickerAccess');
const db = require('~/models');
const { findMCPServerByObjectId, getSkillById } = db;
@@ -198,6 +204,7 @@ router.put(
checkShareAccessUnlessAgentAdmin,
checkSharePublicAccess,
rejectSharedLinkOwnerPermissionChanges,
+ configMiddleware,
updateResourcePermissions,
);
diff --git a/api/server/routes/accessPermissions.sharePolicy.test.js b/api/server/routes/accessPermissions.sharePolicy.test.js
index ed17a04452..b00299da27 100644
--- a/api/server/routes/accessPermissions.sharePolicy.test.js
+++ b/api/server/routes/accessPermissions.sharePolicy.test.js
@@ -9,6 +9,10 @@ jest.mock('~/server/middleware', () => ({
checkBan: (_req, _res, next) => next(),
uaParser: (_req, _res, next) => next(),
canAccessResource: jest.fn(() => (_req, _res, next) => next()),
+ configMiddleware: (req, _res, next) => {
+ req.config = { config: { permissions: { leaseTtlMs: 45000 } } };
+ next();
+ },
}));
jest.mock('~/server/middleware/checkPeoplePickerAccess', () => ({
@@ -207,6 +211,22 @@ describe('Access permissions share policy', () => {
expect(updateResourcePermissions).toHaveBeenCalledTimes(1);
});
+ it('loads the app config before the update so permissions.lease* values reach the controller', async () => {
+ getRoleByName.mockResolvedValue({
+ permissions: {
+ [PermissionTypes.SKILLS]: { [Permissions.SHARE]: true, [Permissions.SHARE_PUBLIC]: false },
+ },
+ });
+
+ await request(app)
+ .put(`/api/permissions/${ResourceType.SKILL}/${resourceId}`)
+ .send({ updated: [createUpdatedPrincipal(AccessRoleIds.SKILL_VIEWER)], public: false });
+
+ expect(updateResourcePermissions.mock.calls[0][0].config.config.permissions.leaseTtlMs).toBe(
+ 45000,
+ );
+ });
+
it('preserves resource management capability bypass for non-public skill sharing', async () => {
hasCapability.mockResolvedValue(true);
diff --git a/api/server/routes/admin/grants.js b/api/server/routes/admin/grants.js
index f48c3ddd60..9986e17229 100644
--- a/api/server/routes/admin/grants.js
+++ b/api/server/routes/admin/grants.js
@@ -1,7 +1,12 @@
const express = require('express');
-const { createAdminGrantsHandlers, getCachedPrincipals } = require('@librechat/api');
+const {
+ createAdminGrantsHandlers,
+ getCachedPrincipals,
+ isAuditFailClosed,
+} = require('@librechat/api');
const { SystemCapabilities } = require('@librechat/data-schemas');
const { requireCapability } = require('~/server/middleware/roles/capabilities');
+const configMiddleware = require('~/server/middleware/config/app');
const { requireJwtAuth } = require('~/server/middleware');
const db = require('~/models');
@@ -22,8 +27,12 @@ const handlers = createAdminGrantsHandlers({
getCachedPrincipals,
checkRoleExists: async (name) => (await db.getRoleByName(name)) != null,
recordAuditEntry: db.recordAuditEntry,
- /** Opt-in: fail the grant request if its audit entry can't be persisted. */
- auditFailClosed: process.env.AUDIT_LOG_FAIL_CLOSED === 'true',
+ auditFailClosed: isAuditFailClosed(),
+ lease: {
+ acquireResourceLock: db.acquireResourceLock,
+ renewResourceLock: db.renewResourceLock,
+ releaseResourceLock: db.releaseResourceLock,
+ },
});
router.use(requireJwtAuth, requireAdminAccess);
@@ -31,8 +40,9 @@ router.use(requireJwtAuth, requireAdminAccess);
router.get('/', handlers.listGrants);
router.get('/effective', handlers.getEffectiveCapabilities);
router.get('/:principalType/:principalId', handlers.getPrincipalGrants);
-router.post('/', handlers.assignGrant);
+/** Only the mutations read `permissions.lease*` from the loaded config. */
+router.post('/', configMiddleware, handlers.assignGrant);
/** Callers should encodeURIComponent the capability for client compatibility (e.g. manage%3Aconfigs%3Aendpoints). */
-router.delete('/:principalType/:principalId/:capability', handlers.revokeGrant);
+router.delete('/:principalType/:principalId/:capability', configMiddleware, handlers.revokeGrant);
module.exports = router;
diff --git a/api/server/services/PermissionService.js b/api/server/services/PermissionService.js
index 98a19a9c71..c525c0a0b8 100644
--- a/api/server/services/PermissionService.js
+++ b/api/server/services/PermissionService.js
@@ -695,9 +695,6 @@ const hasPublicPermission = async ({ resourceType, resourceId, requiredPermissio
const accessControlService = new AccessControlService(mongoose, db);
const bulkUpdateResourcePermissions = (params) =>
accessControlService.bulkUpdateResourcePermissions(params);
-const restoreInsightsPermissionChanges = (params) =>
- accessControlService.restoreInsightsPermissionChanges(params);
-
/**
* Remove all permissions for a resource (cleanup when resource is deleted)
* @param {Object} params - Parameters for removing all permissions
@@ -739,7 +736,6 @@ module.exports = {
hasPublicPermission,
getAvailableRoles,
bulkUpdateResourcePermissions,
- restoreInsightsPermissionChanges,
ensurePrincipalExists,
ensureGroupPrincipalExists,
syncUserEntraGroupMemberships,
diff --git a/api/server/services/PermissionService.spec.js b/api/server/services/PermissionService.spec.js
index 0ab7411493..604b03bfec 100644
--- a/api/server/services/PermissionService.spec.js
+++ b/api/server/services/PermissionService.spec.js
@@ -10,7 +10,6 @@ const {
} = require('librechat-data-provider');
const {
bulkUpdateResourcePermissions,
- restoreInsightsPermissionChanges,
syncUserEntraGroupMemberships,
getEffectivePermissions,
findAccessibleResources,
@@ -1107,260 +1106,6 @@ describe('PermissionService', () => {
},
);
- test('removes a stale-read upsert after a concurrent deletion and audit failure', async () => {
- const originalBulkWriteAclEntries = AclEntry.bulkWrite.bind(AclEntry);
- const bulkWriteSpy = jest
- .spyOn(AclEntry, 'bulkWrite')
- .mockImplementationOnce(async (...args) => {
- await AclEntry.deleteOne({
- principalType: PrincipalType.USER,
- principalId: userId,
- resourceType: ResourceType.AGENT,
- resourceId,
- });
- return originalBulkWriteAclEntries(...args);
- });
-
- let result;
- try {
- result = await bulkUpdateResourcePermissions({
- resourceType: ResourceType.AGENT,
- resourceId,
- updatedPrincipals: [
- {
- type: PrincipalType.USER,
- id: userId,
- accessRoleId: AccessRoleIds.AGENT_EDITOR,
- viewInsights: true,
- },
- ],
- grantedBy: grantedById,
- });
- } finally {
- bulkWriteSpy.mockRestore();
- }
-
- expect(result.insightsChanges[0].previousEntry).toBeNull();
- await restoreInsightsPermissionChanges({
- resourceType: ResourceType.AGENT,
- resourceId,
- changes: result.insightsChanges,
- });
-
- await expect(
- AclEntry.exists({
- principalType: PrincipalType.USER,
- principalId: userId,
- resourceType: ResourceType.AGENT,
- resourceId,
- }),
- ).resolves.toBeNull();
- });
-
- test('restores the prior ACL document after an Insights audit failure', async () => {
- const before = await AclEntry.findOne({
- principalType: PrincipalType.USER,
- principalId: userId,
- resourceType: ResourceType.AGENT,
- resourceId,
- }).lean();
- const result = await bulkUpdateResourcePermissions({
- resourceType: ResourceType.AGENT,
- resourceId,
- updatedPrincipals: [
- {
- type: PrincipalType.USER,
- id: userId,
- accessRoleId: AccessRoleIds.AGENT_EDITOR,
- viewInsights: true,
- },
- ],
- grantedBy: grantedById,
- });
-
- await restoreInsightsPermissionChanges({
- resourceType: ResourceType.AGENT,
- resourceId,
- changes: result.insightsChanges,
- });
-
- const restored = await AclEntry.findById(before._id).lean();
- expect(restored).toEqual(before);
- });
-
- test('does not overwrite or resurrect an ACL document changed after the audited update', async () => {
- const result = await bulkUpdateResourcePermissions({
- resourceType: ResourceType.AGENT,
- resourceId,
- updatedPrincipals: [
- {
- type: PrincipalType.USER,
- id: userId,
- accessRoleId: AccessRoleIds.AGENT_EDITOR,
- viewInsights: true,
- },
- ],
- grantedBy: grantedById,
- });
- const change = result.insightsChanges[0];
- const ownerRole = await findRoleByIdentifier(AccessRoleIds.AGENT_OWNER);
- const newerGrantedAt = new Date(change.writtenEntry.grantedAt.getTime() + 1000);
-
- await AclEntry.updateOne(
- { _id: change.previousEntry._id },
- {
- $set: {
- permBits: ownerRole.permBits,
- roleId: ownerRole._id,
- grantedBy: otherUserId,
- grantedAt: newerGrantedAt,
- },
- },
- );
- await restoreInsightsPermissionChanges({
- resourceType: ResourceType.AGENT,
- resourceId,
- changes: result.insightsChanges,
- });
-
- let current = await AclEntry.findById(change.previousEntry._id).lean();
- expect(current.permBits).toBe(ownerRole.permBits);
- expect(current.grantedAt).toEqual(newerGrantedAt);
-
- await AclEntry.deleteOne({ _id: change.previousEntry._id });
- await restoreInsightsPermissionChanges({
- resourceType: ResourceType.AGENT,
- resourceId,
- changes: result.insightsChanges,
- });
-
- current = await AclEntry.findById(change.previousEntry._id).lean();
- expect(current).toBeNull();
- });
-
- test('removes a new ACL document after an Insights audit failure', async () => {
- const result = await bulkUpdateResourcePermissions({
- resourceType: ResourceType.AGENT,
- resourceId,
- updatedPrincipals: [
- {
- type: PrincipalType.USER,
- id: otherUserId,
- accessRoleId: AccessRoleIds.AGENT_VIEWER,
- viewInsights: true,
- },
- ],
- grantedBy: grantedById,
- });
-
- await restoreInsightsPermissionChanges({
- resourceType: ResourceType.AGENT,
- resourceId,
- changes: result.insightsChanges,
- });
-
- await expect(
- AclEntry.exists({
- principalType: PrincipalType.USER,
- principalId: otherUserId,
- resourceType: ResourceType.AGENT,
- resourceId,
- }),
- ).resolves.toBeNull();
- });
-
- test('does not remove a new ACL document changed after the audited insert', async () => {
- const result = await bulkUpdateResourcePermissions({
- resourceType: ResourceType.AGENT,
- resourceId,
- updatedPrincipals: [
- {
- type: PrincipalType.USER,
- id: otherUserId,
- accessRoleId: AccessRoleIds.AGENT_VIEWER,
- viewInsights: true,
- },
- ],
- grantedBy: grantedById,
- });
- const change = result.insightsChanges[0];
- const editorRole = await findRoleByIdentifier(AccessRoleIds.AGENT_EDITOR);
- const newerGrantedAt = new Date(change.writtenEntry.grantedAt.getTime() + 1000);
-
- await AclEntry.updateOne(
- {
- principalType: PrincipalType.USER,
- principalId: otherUserId,
- resourceType: ResourceType.AGENT,
- resourceId,
- },
- {
- $set: {
- permBits: editorRole.permBits,
- roleId: editorRole._id,
- grantedAt: newerGrantedAt,
- },
- },
- );
- await restoreInsightsPermissionChanges({
- resourceType: ResourceType.AGENT,
- resourceId,
- changes: result.insightsChanges,
- });
-
- const current = await AclEntry.findOne({
- principalType: PrincipalType.USER,
- principalId: otherUserId,
- resourceType: ResourceType.AGENT,
- resourceId,
- }).lean();
- expect(current.permBits).toBe(editorRole.permBits);
- expect(current.grantedAt).toEqual(newerGrantedAt);
- });
-
- test('does not replace a newer ACL grant when restoring an audited deletion', async () => {
- await AclEntry.updateOne(
- {
- principalType: PrincipalType.USER,
- principalId: userId,
- resourceType: ResourceType.AGENT,
- resourceId,
- },
- { $bit: { permBits: { or: PermissionBits.VIEW_INSIGHTS } } },
- );
- const result = await bulkUpdateResourcePermissions({
- resourceType: ResourceType.AGENT,
- resourceId,
- revokedPrincipals: [{ type: PrincipalType.USER, id: userId }],
- grantedBy: grantedById,
- });
-
- await grantPermission({
- principalType: PrincipalType.USER,
- principalId: userId,
- resourceType: ResourceType.AGENT,
- resourceId,
- accessRoleId: AccessRoleIds.AGENT_EDITOR,
- grantedBy: otherUserId,
- });
- await restoreInsightsPermissionChanges({
- resourceType: ResourceType.AGENT,
- resourceId,
- changes: result.insightsChanges,
- });
-
- const entries = await AclEntry.find({
- principalType: PrincipalType.USER,
- principalId: userId,
- resourceType: ResourceType.AGENT,
- resourceId,
- })
- .populate('roleId', 'accessRoleId')
- .lean();
- expect(entries).toHaveLength(1);
- expect(entries[0].roleId.accessRoleId).toBe(AccessRoleIds.AGENT_EDITOR);
- });
-
test('should revoke specified permissions', async () => {
const revokedPrincipals = [
{
diff --git a/client/src/components/Sharing/GenericGrantAccessDialog.tsx b/client/src/components/Sharing/GenericGrantAccessDialog.tsx
index 93503ac6fe..aed8d849b4 100644
--- a/client/src/components/Sharing/GenericGrantAccessDialog.tsx
+++ b/client/src/components/Sharing/GenericGrantAccessDialog.tsx
@@ -33,6 +33,7 @@ import UnifiedPeopleSearch from './PeoplePicker/UnifiedPeopleSearch';
import PeoplePickerAdminSettings from './PeoplePickerAdminSettings';
import PublicSharingToggle from './PublicSharingToggle';
import { SelectedPrincipalsList } from './PeoplePicker';
+import { permissionErrorKey } from './errors';
import { cn } from '~/utils';
export default function GenericGrantAccessDialog({
@@ -220,7 +221,7 @@ export default function GenericGrantAccessDialog({
} catch (error) {
console.error('Error updating permissions:', error);
showToast({
- message: localize('com_ui_permissions_failed_update'),
+ message: localize(permissionErrorKey(error)),
status: 'error',
});
}
diff --git a/client/src/components/Sharing/__tests__/GenericGrantAccessDialog.spec.tsx b/client/src/components/Sharing/__tests__/GenericGrantAccessDialog.spec.tsx
index dbbeff5eaf..5ac534c4ab 100644
--- a/client/src/components/Sharing/__tests__/GenericGrantAccessDialog.spec.tsx
+++ b/client/src/components/Sharing/__tests__/GenericGrantAccessDialog.spec.tsx
@@ -1,9 +1,11 @@
import React from 'react';
import { ResourceType } from 'librechat-data-provider';
-import { render, screen, fireEvent } from '@testing-library/react';
+import { render, screen, fireEvent, waitFor } from '@testing-library/react';
import '@testing-library/jest-dom/extend-expect';
import GenericGrantAccessDialog from '../GenericGrantAccessDialog';
+import translationEn from '~/locales/en/translation.json';
+const mockShowToast = jest.fn();
const mockRefetchPermissions = jest.fn();
const mockUseResourcePermissionState = jest.fn();
const mockUseAuthContext = jest.fn(() => ({ user: { role: 'ADMIN' } }));
@@ -50,7 +52,7 @@ jest.mock('~/hooks', () => ({
jest.mock('@librechat/client', () => ({
...jest.requireActual('@librechat/client'),
- useToastContext: () => ({ showToast: jest.fn() }),
+ useToastContext: () => ({ showToast: mockShowToast }),
}));
jest.mock('../PeoplePicker/UnifiedPeopleSearch', () => ({
@@ -195,4 +197,31 @@ describe('GenericGrantAccessDialog - permissions load failure', () => {
expect(setIsPublic).toHaveBeenCalledWith(false);
expect(setPublicRole).toHaveBeenCalledWith('viewer');
});
+
+ it.each([
+ ['RESOURCE_PERMISSIONS_BUSY', 'com_ui_permissions_busy'],
+ ['RESOURCE_PERMISSIONS_UNAVAILABLE', 'com_ui_permissions_unavailable'],
+ ['PERMISSION_AUDIT_UNAVAILABLE', 'com_ui_permissions_audit_unavailable'],
+ ['SOMETHING_ELSE', 'com_ui_permissions_failed_update'],
+ ])('shows the localized message for the %s failure code on save', async (code, key) => {
+ const mutateAsync = jest.fn().mockRejectedValue({ response: { data: { code } } });
+ mockUseResourcePermissionState.mockReturnValue(
+ baseState({
+ isPublic: true,
+ currentIsPublic: false,
+ updatePermissionsMutation: { isLoading: false, mutateAsync },
+ }),
+ );
+ mockShowToast.mockClear();
+ jest.spyOn(console, 'error').mockImplementation(() => undefined);
+
+ renderDialog();
+ fireEvent.click(screen.getByRole('button', { name: 'com_ui_share_var' }));
+ fireEvent.click(screen.getByRole('button', { name: 'com_ui_save' }));
+
+ await waitFor(() =>
+ expect(mockShowToast).toHaveBeenCalledWith({ message: key, status: 'error' }),
+ );
+ expect((translationEn as Record<string, string>)[key]).toEqual(expect.any(String));
+ });
});
diff --git a/client/src/components/Sharing/__tests__/errors.spec.ts b/client/src/components/Sharing/__tests__/errors.spec.ts
new file mode 100644
index 0000000000..c5fe1ddd13
--- /dev/null
+++ b/client/src/components/Sharing/__tests__/errors.spec.ts
@@ -0,0 +1,20 @@
+import { permissionErrorKey } from '../errors';
+
+const withCode = (code: string) => ({ response: { data: { code } } });
+
+describe('permissionErrorKey', () => {
+ it.each([
+ ['RESOURCE_PERMISSIONS_BUSY', 'com_ui_permissions_busy'],
+ ['RESOURCE_PERMISSIONS_UNAVAILABLE', 'com_ui_permissions_unavailable'],
+ ['PERMISSION_AUDIT_UNAVAILABLE', 'com_ui_permissions_audit_unavailable'],
+ ])('maps %s to an actionable message', (code, key) => {
+ expect(permissionErrorKey(withCode(code))).toBe(key);
+ });
+
+ it.each([withCode('SOMETHING_ELSE'), new Error('x'), null, undefined])(
+ 'falls back to the generic failure for %p',
+ (error) => {
+ expect(permissionErrorKey(error)).toBe('com_ui_permissions_failed_update');
+ },
+ );
+});
diff --git a/client/src/components/Sharing/errors.ts b/client/src/components/Sharing/errors.ts
new file mode 100644
index 0000000000..624831f3e9
--- /dev/null
+++ b/client/src/components/Sharing/errors.ts
@@ -0,0 +1,13 @@
+import type { TranslationKeys } from '~/hooks';
+
+const PERMISSION_ERROR_KEYS: Record<string, TranslationKeys> = {
+ RESOURCE_PERMISSIONS_BUSY: 'com_ui_permissions_busy',
+ RESOURCE_PERMISSIONS_UNAVAILABLE: 'com_ui_permissions_unavailable',
+ PERMISSION_AUDIT_UNAVAILABLE: 'com_ui_permissions_audit_unavailable',
+};
+
+/** The localized message for a failed permission update, from the stable code the API returns. */
+export function permissionErrorKey(error: unknown): TranslationKeys {
+ const code = (error as { response?: { data?: { code?: string } } } | null)?.response?.data?.code;
+ return (code && PERMISSION_ERROR_KEYS[code]) || 'com_ui_permissions_failed_update';
+}
diff --git a/client/src/locales/en/translation.json b/client/src/locales/en/translation.json
index d6c148b2a4..6cd25eb8c5 100644
--- a/client/src/locales/en/translation.json
+++ b/client/src/locales/en/translation.json
@@ -2468,8 +2468,11 @@
"com_ui_people_picker_allow_view_groups": "Allow viewing groups",
"com_ui_people_picker_allow_view_roles": "Allow viewing roles",
"com_ui_people_picker_allow_view_users": "Allow viewing users",
+ "com_ui_permissions_audit_unavailable": "Permissions could not be updated because the audit log is unavailable. Please try again shortly.",
+ "com_ui_permissions_busy": "Another change to these permissions is in progress. Please try again in a moment.",
"com_ui_permissions_failed_load": "Failed to load permissions. Please try again.",
"com_ui_permissions_failed_update": "Failed to update permissions. Please try again.",
+ "com_ui_permissions_unavailable": "Permissions cannot be updated right now. Please try again shortly.",
"com_ui_permissions_updated_success": "Permissions updated successfully",
"com_ui_schedules_filter": "Filter schedules...",
"com_ui_sharing": "Sharing",
diff --git a/docs/permissions/acl-write-compatibility.md b/docs/permissions/acl-write-compatibility.md
index 89655dd1e1..13e0b18ee9 100644
--- a/docs/permissions/acl-write-compatibility.md
+++ b/docs/permissions/acl-write-compatibility.md
@@ -10,8 +10,17 @@ masks. Neither path uses MongoDB bit-update operators.
```yaml
permissions:
maxWriteAttempts: 3
+ leaseTtlMs: 30000
+ leaseWaitMs: 3000
```
+`leaseTtlMs` (1000 to 600000) and `leaseWaitMs` (0 to 60000) apply only where transactions are
+unavailable: sharing updates and system grant changes then take a per-resource or per-grant lease,
+renewed while the update runs, and answer 409 when it stays contended past `leaseWaitMs`. The lease only
+coordinates instances that run this version, so during a rolling upgrade of a deployment without
+transactions an older instance can still write without it; finish the upgrade before relying on the
+ordering between audit entries and ACL writes.
+
The sharing endpoint passes this setting to the persistence operation. The value must be an integer
from 1 through 100, including the initial attempt. Internal callers can override the same validated
budget per call. A session alone does not imply a transaction: comparison reads use primary unless
diff --git a/e2e/specs/mock/scenarios/resource-acl-audit.spec.ts b/e2e/specs/mock/scenarios/resource-acl-audit.spec.ts
new file mode 100644
index 0000000000..60c3d762ee
--- /dev/null
+++ b/e2e/specs/mock/scenarios/resource-acl-audit.spec.ts
@@ -0,0 +1,137 @@
+import { randomUUID } from 'crypto';
+import { ObjectId } from 'mongodb';
+import { expect, test } from '@playwright/test';
+import type { Page } from '@playwright/test';
+import { getAccessToken, requestJson } from '../helpers';
+import { withMongo } from '../db';
+
+/**
+ * Sharing a resource writes one `acl.*` audit entry per effective grant and
+ * revoke. The entries name the principal by id and carry permission bits only;
+ * display names and emails of the people involved never reach the log.
+ */
+
+const RESOURCE_TYPE = 'promptGroup';
+const VIEWER_ROLE = 'promptGroup_viewer';
+
+type CreatedGroup = { group?: { _id: string; name: string } };
+type AuditRow = { action: string; target?: { id?: string }; metadata?: Record<string, unknown> };
+
+async function createGroup(page: Page, name: string): Promise<string> {
+ const token = await getAccessToken(page);
+ const body = await requestJson<CreatedGroup>(page, {
+ path: '/api/prompts',
+ token,
+ method: 'POST',
+ body: { prompt: { prompt: `Text for ${name}`, type: 'text' }, group: { name } },
+ });
+ const id = body.group?._id ?? '';
+ expect(id).not.toBe('');
+ return id;
+}
+
+async function seedUser(suffix: string) {
+ const user = {
+ _id: new ObjectId(),
+ email: `acl-audit-${suffix}@example.com`,
+ name: `Acl Audit ${suffix}`,
+ };
+ await withMongo((db) => db.collection('users').insertOne(user));
+ return user;
+}
+
+const auditRows = (groupId: string, prefix: string) =>
+ withMongo((db) =>
+ db
+ .collection('auditlogs')
+ .find({ 'target.id': groupId, action: { $regex: `^${prefix}` } })
+ .sort({ seq: 1 })
+ .toArray(),
+ ) as Promise<AuditRow[]>;
+
+async function share(page: Page, groupId: string, body: unknown) {
+ const token = await getAccessToken(page);
+ await requestJson<unknown>(page, {
+ path: `/api/permissions/${RESOURCE_TYPE}/${groupId}`,
+ token,
+ method: 'PUT',
+ body,
+ });
+}
+
+test.describe('resource ACL audit', () => {
+ test('sharing then unsharing a prompt group records a grant and a revoke by principal id @scenario:sharing-a-resource-records-acl-grant-and-revoke-by-principal-id', async ({
+ page,
+ }) => {
+ await page.goto('/c/new');
+ const suffix = randomUUID().slice(0, 8);
+ const groupId = await createGroup(page, `Acl audit ${suffix}`);
+ const target = await seedUser(suffix);
+ const principal = {
+ type: 'user',
+ id: target._id.toString(),
+ name: target.name,
+ email: target.email,
+ source: 'local',
+ };
+
+ try {
+ await share(page, groupId, {
+ updated: [{ ...principal, accessRoleId: VIEWER_ROLE }],
+ removed: [],
+ });
+ await expect
+ .poll(async () => (await auditRows(groupId, 'acl.')).map((row) => row.action))
+ .toEqual(['acl.granted']);
+
+ await share(page, groupId, { updated: [], removed: [principal] });
+ await expect
+ .poll(async () => (await auditRows(groupId, 'acl.')).map((row) => row.action))
+ .toEqual(['acl.granted', 'acl.revoked']);
+
+ const rows = await auditRows(groupId, 'acl.');
+ expect(rows[0].metadata).toEqual(
+ expect.objectContaining({ principalType: 'user', principalId: principal.id }),
+ );
+ expect(rows[1].metadata).toEqual(
+ expect.objectContaining({ principalType: 'user', principalId: principal.id }),
+ );
+ const serialized = JSON.stringify(rows);
+ expect(serialized).not.toContain(target.name);
+ expect(serialized).not.toContain(target.email);
+ } finally {
+ await withMongo((db) => db.collection('users').deleteOne({ _id: target._id }));
+ }
+ });
+
+ test('re-sharing with the same role writes no new audit entry @scenario:resharing-with-an-unchanged-role-records-no-acl-entry', async ({
+ page,
+ }) => {
+ await page.goto('/c/new');
+ const suffix = randomUUID().slice(0, 8);
+ const groupId = await createGroup(page, `Acl noop ${suffix}`);
+ const target = await seedUser(suffix);
+ const body = {
+ updated: [
+ {
+ type: 'user',
+ id: target._id.toString(),
+ name: target.name,
+ email: target.email,
+ source: 'local',
+ accessRoleId: VIEWER_ROLE,
+ },
+ ],
+ removed: [],
+ };
+
+ try {
+ await share(page, groupId, body);
+ await expect.poll(async () => (await auditRows(groupId, 'acl.')).length).toBe(1);
+ await share(page, groupId, body);
+ expect(await auditRows(groupId, 'acl.')).toHaveLength(1);
+ } finally {
+ await withMongo((db) => db.collection('users').deleteOne({ _id: target._id }));
+ }
+ });
+});
diff --git a/librechat.example.yaml b/librechat.example.yaml
index fb8b4a399c..f00bd479fa 100644
--- a/librechat.example.yaml
+++ b/librechat.example.yaml
@@ -1749,3 +1749,5 @@ endpoints:
# ACL writes retry only conflicts on an individual document.
# permissions:
# maxWriteAttempts: 3 # Integer 1-100, including the first attempt.
+# leaseTtlMs: 30000 # Integer 1000-600000. Sharing updates on deployments without transactions hold a per-resource lease this long between renewals.
+# leaseWaitMs: 3000 # Integer 0-60000. How long an update waits for a contended lease before answering 409.
diff --git a/packages/api/src/acl/accessControlService.spec.ts b/packages/api/src/acl/accessControlService.spec.ts
index 474ad33aec..c5113eafba 100644
--- a/packages/api/src/acl/accessControlService.spec.ts
+++ b/packages/api/src/acl/accessControlService.spec.ts
@@ -9,7 +9,11 @@ import {
PrincipalType,
} from 'librechat-data-provider';
import type { IAclEntry } from '@librechat/data-schemas';
-import { AccessControlService } from './accessControlService';
+import {
+ AccessControlService,
+ permissionErrorCode,
+ ResourcePermissionsBusyError,
+} from './accessControlService';
// Mock the logger
jest.mock('@librechat/data-schemas', () => ({
@@ -74,7 +78,7 @@ describe('AccessControlService', () => {
const grantedById = new Types.ObjectId();
describe('bulkUpdateResourcePermissions', () => {
- test('records only the revocation that actually deleted Insights access', async () => {
+ test('plans the Insights revocation per request and leaves the access removed', async () => {
await service.bulkUpdateResourcePermissions({
resourceType: ResourceType.AGENT,
resourceId,
@@ -98,9 +102,10 @@ describe('AccessControlService', () => {
});
const results = await Promise.all([revoke(), revoke()]);
- expect(results.flatMap((result) => result.insightsChanges)).toEqual([
- expect.objectContaining({ action: 'removed' }),
- ]);
+ /** Audit precedes the write: a racing request plans the revocation it read, or none if it read after the delete. */
+ const planned = results.flatMap((result) => result.insightsChanges);
+ expect(planned.length).toBeGreaterThanOrEqual(1);
+ expect(planned.every((change) => change.action === 'removed')).toBe(true);
await expect(
AclEntry.exists({
principalType: PrincipalType.USER,
@@ -205,59 +210,6 @@ describe('AccessControlService', () => {
).toBe(true);
});
- test('removes Insights from every duplicate when an assignment is rolled back', async () => {
- await service.bulkUpdateResourcePermissions({
- resourceType: ResourceType.AGENT,
- resourceId,
- updatedPrincipals: [
- {
- type: PrincipalType.USER,
- id: userId.toString(),
- accessRoleId: AccessRoleIds.AGENT_VIEWER,
- viewInsights: false,
- },
- ],
- grantedBy: grantedById,
- });
- const entry = await AclEntry.findOne({
- principalType: PrincipalType.USER,
- principalId: userId,
- resourceType: ResourceType.AGENT,
- resourceId,
- }).lean();
- await AclEntry.create({ ...entry, _id: new Types.ObjectId() });
-
- const result = await service.bulkUpdateResourcePermissions({
- resourceType: ResourceType.AGENT,
- resourceId,
- updatedPrincipals: [
- {
- type: PrincipalType.USER,
- id: userId.toString(),
- accessRoleId: AccessRoleIds.AGENT_VIEWER,
- viewInsights: true,
- },
- ],
- grantedBy: grantedById,
- });
- await service.restoreInsightsPermissionChanges({
- resourceType: ResourceType.AGENT,
- resourceId,
- changes: result.insightsChanges,
- });
-
- const entries = await AclEntry.find({
- principalType: PrincipalType.USER,
- principalId: userId,
- resourceType: ResourceType.AGENT,
- resourceId,
- }).lean();
- expect(entries).toHaveLength(2);
- expect(
- entries.every((candidate) => (candidate.permBits & PermissionBits.VIEW_INSIGHTS) === 0),
- ).toBe(true);
- });
-
test('audits a revoke when any duplicate ACL entry has Insights access', async () => {
const viewerRole = await dbMethods.findRoleByIdentifier(AccessRoleIds.AGENT_VIEWER);
await AclEntry.create([
@@ -295,13 +247,500 @@ describe('AccessControlService', () => {
expect(result.insightsChanges).toEqual([
expect.objectContaining({
action: 'removed',
- previousEntry: expect.objectContaining({
+ previous: expect.objectContaining({
permBits: PermissionBits.VIEW | PermissionBits.VIEW_INSIGHTS,
}),
}),
]);
});
+ test('reports every effective ACL change as plain ids and bits', async () => {
+ const principal = { type: PrincipalType.USER, id: userId.toString() };
+ const grant = (accessRoleId: AccessRoleIds) =>
+ service.bulkUpdateResourcePermissions({
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId,
+ updatedPrincipals: [{ ...principal, accessRoleId }],
+ grantedBy: grantedById,
+ });
+ const entryBits = async () =>
+ (
+ await AclEntry.findOne({
+ principalId: userId,
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId,
+ })
+ )?.permBits;
+
+ const created = await grant(AccessRoleIds.PROMPTGROUP_VIEWER);
+ expect(created.insightsChanges).toEqual([]);
+ expect(created.aclChanges).toEqual([
+ { action: 'assigned', previous: null, written: expect.any(Object), principal },
+ ]);
+ const viewerBits = await entryBits();
+
+ const updated = await grant(AccessRoleIds.PROMPTGROUP_EDITOR);
+ expect(updated.aclChanges).toEqual([
+ expect.objectContaining({
+ action: 'assigned',
+ previous: { permBits: viewerBits, roleId: expect.any(String) },
+ written: { permBits: await entryBits(), roleId: expect.any(String) },
+ }),
+ ]);
+
+ const revoked = await service.bulkUpdateResourcePermissions({
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId,
+ revokedPrincipals: [principal],
+ grantedBy: grantedById,
+ });
+ expect(revoked.aclChanges).toEqual([
+ expect.objectContaining({ action: 'removed', written: null }),
+ ]);
+ expect(await entryBits()).toBeUndefined();
+
+ const noop = await service.bulkUpdateResourcePermissions({
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId,
+ revokedPrincipals: [{ type: PrincipalType.USER, id: new Types.ObjectId().toString() }],
+ grantedBy: grantedById,
+ });
+ expect(noop.aclChanges).toEqual([]);
+ });
+
+ test('calls beforeWrite with the changes before any ACL write, and a throw leaves the ACL untouched', async () => {
+ const seen: unknown[] = [];
+ const entryCount = () =>
+ AclEntry.countDocuments({ resourceType: ResourceType.PROMPTGROUP, resourceId });
+
+ await expect(
+ service.bulkUpdateResourcePermissions({
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId,
+ updatedPrincipals: [
+ {
+ type: PrincipalType.USER,
+ id: userId.toString(),
+ accessRoleId: AccessRoleIds.PROMPTGROUP_VIEWER,
+ },
+ ],
+ grantedBy: grantedById,
+ beforeWrite: async (plan) => {
+ seen.push(plan.aclChanges.length, await entryCount());
+ throw new Error('audit unavailable');
+ },
+ }),
+ ).rejects.toThrow('audit unavailable');
+
+ expect(seen).toEqual([1, 0]);
+ expect(await entryCount()).toBe(0);
+ });
+
+ test('a revoke with nothing audited does not delete an entry that appeared after the read', async () => {
+ const viewerRole = await dbMethods.findRoleByIdentifier(AccessRoleIds.PROMPTGROUP_VIEWER);
+
+ const result = await service.bulkUpdateResourcePermissions({
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId,
+ revokedPrincipals: [{ type: PrincipalType.USER, id: userId.toString() }],
+ grantedBy: grantedById,
+ beforeWrite: async () => {
+ throw new Error('beforeWrite must not run for an empty plan');
+ },
+ });
+ expect(result.aclChanges).toEqual([]);
+
+ const entry = {
+ principalType: PrincipalType.USER,
+ principalId: userId,
+ principalModel: PrincipalModel.USER,
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId,
+ permBits: PermissionBits.VIEW,
+ roleId: viewerRole?._id,
+ grantedBy: grantedById,
+ grantedAt: new Date(),
+ };
+ const spy = jest
+ .spyOn(service['_dbMethods'], 'findEntriesByPrincipalsAndResource')
+ .mockImplementationOnce(async () => {
+ await AclEntry.create(entry);
+ return [];
+ });
+ await service.bulkUpdateResourcePermissions({
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId,
+ revokedPrincipals: [{ type: PrincipalType.USER, id: userId.toString() }],
+ grantedBy: grantedById,
+ });
+ spy.mockRestore();
+
+ await expect(
+ AclEntry.exists({
+ principalId: userId,
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId,
+ }),
+ ).resolves.not.toBeNull();
+ });
+
+ test('audits a revoke whose principal id differs only in case from the stored id', async () => {
+ const viewerRole = await dbMethods.findRoleByIdentifier(AccessRoleIds.PROMPTGROUP_VIEWER);
+ await AclEntry.create({
+ principalType: PrincipalType.USER,
+ principalId: userId,
+ principalModel: PrincipalModel.USER,
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId,
+ permBits: PermissionBits.VIEW,
+ roleId: viewerRole?._id,
+ grantedBy: grantedById,
+ grantedAt: new Date(),
+ });
+ const planned: number[] = [];
+
+ const result = await service.bulkUpdateResourcePermissions({
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId,
+ revokedPrincipals: [{ type: PrincipalType.USER, id: userId.toString().toUpperCase() }],
+ grantedBy: grantedById,
+ beforeWrite: async (plan) => {
+ planned.push(plan.aclChanges.length);
+ },
+ });
+
+ expect(planned).toEqual([1]);
+ expect(result.aclChanges).toEqual([expect.objectContaining({ action: 'removed' })]);
+ await expect(
+ AclEntry.exists({
+ principalId: userId,
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId,
+ }),
+ ).resolves.toBeNull();
+ });
+
+ test('an explicit public disable wins over a public update in the same request', async () => {
+ const result = await service.bulkUpdateResourcePermissions({
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId,
+ updatedPrincipals: [
+ { type: PrincipalType.PUBLIC, accessRoleId: AccessRoleIds.PROMPTGROUP_VIEWER },
+ ],
+ revokedPrincipals: [{ type: PrincipalType.PUBLIC }],
+ grantedBy: grantedById,
+ });
+
+ expect(result.aclChanges).toEqual([]);
+ await expect(
+ AclEntry.exists({
+ principalType: PrincipalType.PUBLIC,
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId,
+ }),
+ ).resolves.toBeNull();
+ });
+
+ test('audits duplicate ACL rows as their combined permissions and ignores Insights on non-agents', async () => {
+ const viewerRole = await dbMethods.findRoleByIdentifier(AccessRoleIds.PROMPTGROUP_VIEWER);
+ const row = (permBits: number) => ({
+ principalType: PrincipalType.USER,
+ principalId: userId,
+ principalModel: PrincipalModel.USER,
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId,
+ permBits,
+ roleId: viewerRole?._id,
+ grantedBy: grantedById,
+ grantedAt: new Date(),
+ });
+ await AclEntry.create([
+ row(PermissionBits.VIEW | PermissionBits.VIEW_INSIGHTS),
+ row(PermissionBits.VIEW | PermissionBits.EDIT),
+ ]);
+
+ const result = await service.bulkUpdateResourcePermissions({
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId,
+ revokedPrincipals: [{ type: PrincipalType.USER, id: userId.toString() }],
+ grantedBy: grantedById,
+ });
+
+ expect(result.insightsChanges).toEqual([]);
+ expect(result.aclChanges).toEqual([
+ expect.objectContaining({
+ previous: expect.objectContaining({
+ permBits: PermissionBits.VIEW | PermissionBits.EDIT | PermissionBits.VIEW_INSIGHTS,
+ }),
+ }),
+ ]);
+ });
+
+ test('plans one revocation when the same principal is revoked twice', async () => {
+ const viewerRole = await dbMethods.findRoleByIdentifier(AccessRoleIds.PROMPTGROUP_VIEWER);
+ await AclEntry.create({
+ principalType: PrincipalType.USER,
+ principalId: userId,
+ principalModel: PrincipalModel.USER,
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId,
+ permBits: PermissionBits.VIEW,
+ roleId: viewerRole?._id,
+ grantedBy: grantedById,
+ grantedAt: new Date(),
+ });
+ const principal = { type: PrincipalType.USER, id: userId.toString() };
+
+ const result = await service.bulkUpdateResourcePermissions({
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId,
+ revokedPrincipals: [principal, { ...principal, id: principal.id.toUpperCase() }],
+ grantedBy: grantedById,
+ });
+
+ expect(result.aclChanges).toHaveLength(1);
+ });
+
+ test('serializes concurrent updates of one resource so each audit precedes its own write', async () => {
+ const events: string[] = [];
+ const acquire = jest.spyOn(service['_dbMethods'], 'acquireResourceLock');
+ const update = (label: string, id: Types.ObjectId, hold: number) =>
+ service
+ .bulkUpdateResourcePermissions({
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId,
+ updatedPrincipals: [
+ {
+ type: PrincipalType.USER,
+ id: id.toString(),
+ accessRoleId: AccessRoleIds.PROMPTGROUP_VIEWER,
+ },
+ ],
+ grantedBy: grantedById,
+ beforeWrite: async () => {
+ events.push(`${label}:audit`);
+ await new Promise((resolve) => setTimeout(resolve, hold));
+ },
+ })
+ .then(() => events.push(`${label}:done`));
+
+ await Promise.all([update('A', userId, 150), update('B', new Types.ObjectId(), 0)]);
+
+ const first = events[0].split(':')[0];
+ expect(events.slice(0, 2)).toEqual([`${first}:audit`, `${first}:done`]);
+ expect(events).toHaveLength(4);
+ expect(acquire.mock.calls.length).toBeGreaterThanOrEqual(2);
+ acquire.mockRestore();
+ });
+
+ test('runs afterWrite even when the post-commit prompt invalidation throws', async () => {
+ const invalidate = jest
+ .spyOn(service['_dbMethods'], 'invalidatePromptGroupAccessContext')
+ .mockRejectedValue(new Error('cache down'));
+ const afterWrite = jest.fn().mockResolvedValue(undefined);
+
+ await expect(
+ service.bulkUpdateResourcePermissions({
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId,
+ updatedPrincipals: [],
+ grantedBy: grantedById,
+ afterWrite,
+ }),
+ ).rejects.toThrow('cache down');
+
+ expect(afterWrite).toHaveBeenCalledTimes(1);
+ invalidate.mockRestore();
+ });
+
+ test('answers a retryable 503 when the lock store fails', async () => {
+ const acquire = jest
+ .spyOn(service['_dbMethods'], 'acquireResourceLock')
+ .mockRejectedValue(new Error('mongo secret detail'));
+
+ await expect(
+ service.bulkUpdateResourcePermissions({
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId,
+ updatedPrincipals: [],
+ grantedBy: grantedById,
+ }),
+ ).rejects.toMatchObject({
+ statusCode: 503,
+ code: 'RESOURCE_PERMISSIONS_UNAVAILABLE',
+ message: expect.not.stringContaining('secret'),
+ });
+ acquire.mockRestore();
+ });
+
+ test('answers a retryable 503 when the lease renewal fails', async () => {
+ const renew = jest
+ .spyOn(service['_dbMethods'], 'renewResourceLock')
+ .mockRejectedValue(new Error('mongo secret detail'));
+
+ await expect(
+ service.bulkUpdateResourcePermissions({
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId,
+ updatedPrincipals: [],
+ grantedBy: grantedById,
+ maxWriteAttempts: 1,
+ beforeWrite: async () => undefined,
+ }),
+ ).rejects.toMatchObject({ statusCode: 503, message: expect.not.stringContaining('secret') });
+ renew.mockRestore();
+ });
+
+ test('honors the configured lease wait and TTL', async () => {
+ const key = `acl:${ResourceType.PROMPTGROUP.toLowerCase()}:${resourceId}`;
+ await service['_dbMethods'].acquireResourceLock({ key, ownerId: 'other', ttlMs: 60_000 });
+ const started = Date.now();
+
+ await expect(
+ service.bulkUpdateResourcePermissions({
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId,
+ updatedPrincipals: [],
+ grantedBy: grantedById,
+ leaseWaitMs: 0,
+ }),
+ ).rejects.toMatchObject({ code: 'RESOURCE_PERMISSIONS_BUSY' });
+ expect(Date.now() - started).toBeLessThan(1000);
+
+ await service['_dbMethods'].releaseResourceLock({ key, ownerId: 'other' });
+ const acquire = jest.spyOn(service['_dbMethods'], 'acquireResourceLock');
+ await service.bulkUpdateResourcePermissions({
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId,
+ updatedPrincipals: [],
+ grantedBy: grantedById,
+ leaseTtlMs: 12_345,
+ });
+ expect(acquire).toHaveBeenCalledWith(expect.objectContaining({ ttlMs: 12_345 }));
+ acquire.mockRestore();
+ });
+
+ test('exposes the busy code for the HTTP error body only', () => {
+ expect(permissionErrorCode(new ResourcePermissionsBusyError())).toEqual({
+ code: 'RESOURCE_PERMISSIONS_BUSY',
+ });
+ expect(
+ permissionErrorCode(
+ Object.assign(new Error('x'), { code: 'PERMISSION_AUDIT_UNAVAILABLE' }),
+ ),
+ ).toEqual({
+ code: 'PERMISSION_AUDIT_UNAVAILABLE',
+ });
+ expect(
+ permissionErrorCode(Object.assign(new Error('x'), { code: 'EOTHER' })),
+ ).toBeUndefined();
+ expect(permissionErrorCode(new Error('x'))).toBeUndefined();
+ });
+
+ test('keys the lease on the canonical resource id', async () => {
+ const acquire = jest.spyOn(service['_dbMethods'], 'acquireResourceLock');
+ for (const id of [resourceId.toString(), resourceId.toString().toUpperCase()]) {
+ await service.bulkUpdateResourcePermissions({
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId: id,
+ updatedPrincipals: [],
+ grantedBy: grantedById,
+ });
+ }
+ const keys = acquire.mock.calls.map(([data]) => data.key);
+ expect(new Set(keys).size).toBe(1);
+ acquire.mockRestore();
+ });
+
+ test('refuses to write once the lease was taken over', async () => {
+ const key = `acl:${ResourceType.PROMPTGROUP.toLowerCase()}:${resourceId}`;
+ await expect(
+ service.bulkUpdateResourcePermissions({
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId,
+ updatedPrincipals: [
+ {
+ type: PrincipalType.USER,
+ id: userId.toString(),
+ accessRoleId: AccessRoleIds.PROMPTGROUP_VIEWER,
+ },
+ ],
+ grantedBy: grantedById,
+ beforeWrite: async () => {
+ await mongoose.models.ResourceLock.updateOne(
+ { key },
+ { $set: { expiresAt: new Date(Date.now() - 1) } },
+ );
+ await service['_dbMethods'].acquireResourceLock({
+ key,
+ ownerId: 'taker',
+ ttlMs: 60_000,
+ });
+ },
+ }),
+ ).rejects.toMatchObject({ code: 'RESOURCE_PERMISSIONS_BUSY' });
+ await expect(
+ AclEntry.exists({
+ principalId: userId,
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId,
+ }),
+ ).resolves.toBeNull();
+ await service['_dbMethods'].releaseResourceLock({ key, ownerId: 'taker' });
+ });
+
+ test('reports a conflict when the resource lease cannot be taken in time', async () => {
+ const key = `acl:${ResourceType.PROMPTGROUP.toLowerCase()}:${resourceId}`;
+ await service['_dbMethods'].acquireResourceLock({ key, ownerId: 'other', ttlMs: 60_000 });
+
+ await expect(
+ service.bulkUpdateResourcePermissions({
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId,
+ updatedPrincipals: [
+ {
+ type: PrincipalType.USER,
+ id: userId.toString(),
+ accessRoleId: AccessRoleIds.PROMPTGROUP_VIEWER,
+ },
+ ],
+ grantedBy: grantedById,
+ }),
+ ).rejects.toMatchObject({ statusCode: 409, code: 'RESOURCE_PERMISSIONS_BUSY' });
+ await service['_dbMethods'].releaseResourceLock({ key, ownerId: 'other' });
+ }, 15000);
+
+ test('revoking snapshots the principal without loading the rest of the ACL', async () => {
+ const otherUserId = new Types.ObjectId();
+ const viewerRole = await dbMethods.findRoleByIdentifier(AccessRoleIds.PROMPTGROUP_VIEWER);
+ for (const principalId of [userId, otherUserId]) {
+ await AclEntry.create({
+ principalType: PrincipalType.USER,
+ principalId,
+ principalModel: PrincipalModel.USER,
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId,
+ permBits: PermissionBits.VIEW,
+ roleId: viewerRole?._id,
+ grantedBy: grantedById,
+ grantedAt: new Date(),
+ });
+ }
+
+ const result = await service.bulkUpdateResourcePermissions({
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId,
+ revokedPrincipals: [{ type: PrincipalType.USER, id: userId.toString() }],
+ grantedBy: grantedById,
+ });
+
+ expect(result.aclChanges).toHaveLength(1);
+ expect(result.aclChanges[0].principal.id).toBe(userId.toString());
+ await expect(
+ AclEntry.exists({ principalId: otherUserId, resourceType: ResourceType.PROMPTGROUP }),
+ ).resolves.not.toBeNull();
+ });
+
test('does not treat a non-agent permission bit as an Insights revocation', async () => {
await AclEntry.create({
principalType: PrincipalType.USER,
diff --git a/packages/api/src/acl/accessControlService.ts b/packages/api/src/acl/accessControlService.ts
index 0f52e845c8..2fbbc4b612 100644
--- a/packages/api/src/acl/accessControlService.ts
+++ b/packages/api/src/acl/accessControlService.ts
@@ -16,8 +16,11 @@ import {
import type { AllMethods, IAclEntry } from '@librechat/data-schemas';
import type { ClientSession, DeleteResult } from 'mongoose';
import type { TPrincipal } from 'librechat-data-provider';
-import type { InsightsPermissionChange, InsightsWrittenEntry } from './insightsPermissions';
+import type { PermissionChange, PermissionSnapshot } from './types';
import type { ResolvedPrincipal } from '~/types/principal';
+import type { PermissionPlan } from './audit';
+import { PERMISSION_AUDIT_UNAVAILABLE, RESOURCE_PERMISSIONS_UNAVAILABLE } from './codes';
+import { getSafeErrorMetadata } from '~/utils/errors';
import { userPrincipalsCache } from '~/cache';
type BulkPrincipal = Omit<TPrincipal, 'id'> & {
@@ -25,11 +28,74 @@ type BulkPrincipal = Omit<TPrincipal, 'id'> & {
memberIds?: string[];
};
+const ROLE_MASK =
+ PermissionBits.VIEW | PermissionBits.EDIT | PermissionBits.DELETE | PermissionBits.SHARE;
+
+/**
+ * One key for a principal, shared by the audit snapshot and the write filter so
+ * both see the same row: ObjectId hex is case-folded and the public principal
+ * has no id.
+ */
+const principalIdentity = (type: string, id: unknown): string => {
+ if (type === PrincipalType.PUBLIC) {
+ return `${type}:null`;
+ }
+ const raw = id == null ? '' : id.toString();
+ const normalized =
+ type !== PrincipalType.ROLE && Types.ObjectId.isValid(raw)
+ ? new Types.ObjectId(raw).toString()
+ : raw;
+ return `${type}:${normalized}`;
+};
+
+/** A holder that crashes frees its lock after this long. */
+const RESOURCE_LOCK_TTL_MS = 30_000;
+const RESOURCE_LOCK_WAIT_MS = 3_000;
+const RESOURCE_LOCK_POLL_MS = 50;
+
+/** The lock store failed; a safe, retryable 503 instead of a raw driver error. */
+export class ResourcePermissionsUnavailableError extends Error {
+ readonly statusCode = 503;
+ readonly code: string = RESOURCE_PERMISSIONS_UNAVAILABLE;
+
+ constructor() {
+ super('Permissions for this resource cannot be updated right now, try again');
+ }
+}
+
+export class ResourcePermissionsBusyError extends Error {
+ readonly statusCode = 409;
+ readonly code = 'RESOURCE_PERMISSIONS_BUSY';
+
+ constructor() {
+ super('Permissions for this resource are being updated, try again');
+ }
+}
+
+const APPROVED_ERROR_CODES = new Set([
+ 'RESOURCE_PERMISSIONS_BUSY',
+ PERMISSION_AUDIT_UNAVAILABLE,
+ RESOURCE_PERMISSIONS_UNAVAILABLE,
+]);
+
+/** The approved machine-readable code of a failed permission update, for the HTTP error body. */
+export const permissionErrorCode = (error: unknown): { code: string } | undefined => {
+ const code = (error as { code?: unknown } | null)?.code;
+ return typeof code === 'string' && APPROVED_ERROR_CODES.has(code) ? { code } : undefined;
+};
+
+const snapshotOf = (
+ entry?: Pick<IAclEntry, 'permBits' | 'roleId'> | null,
+): PermissionSnapshot | null =>
+ entry ? { permBits: entry.permBits, roleId: entry.roleId?.toString() ?? '' } : null;
+
export type BulkPermissionUpdateResult = {
granted: BulkPrincipal[];
updated: BulkPrincipal[];
revoked: BulkPrincipal[];
- insightsChanges: InsightsPermissionChange[];
+ insightsChanges: PermissionChange[];
+ /** Every grant, update and revoke, as the state before and after the write. */
+ aclChanges: PermissionChange[];
errors: Array<{ principal: BulkPrincipal; error: string }>;
};
@@ -411,6 +477,10 @@ export class AccessControlService {
grantedBy,
session,
maxWriteAttempts,
+ leaseTtlMs = RESOURCE_LOCK_TTL_MS,
+ leaseWaitMs = RESOURCE_LOCK_WAIT_MS,
+ beforeWrite,
+ afterWrite,
}: {
resourceType: ResourceType;
resourceId: string | Types.ObjectId;
@@ -419,6 +489,14 @@ export class AccessControlService {
grantedBy: string | Types.ObjectId;
session?: ClientSession;
maxWriteAttempts?: number;
+ /** Per-resource lease lifetime between renewals, from `permissions.leaseTtlMs`. */
+ leaseTtlMs?: number;
+ /** Longest wait for a contended lease, from `permissions.leaseWaitMs`. */
+ leaseWaitMs?: number;
+ /** Called with the computed changes before any ACL write; a throw leaves the ACL untouched. */
+ beforeWrite?: (plan: PermissionPlan) => Promise<void>;
+ /** Runs after the write while the resource lease is still held; its failure never fails the update. */
+ afterWrite?: () => Promise<void>;
}): Promise<BulkPermissionUpdateResult> {
const supportsTransactions = await getTransactionSupport(
this._mongoose,
@@ -427,6 +505,18 @@ export class AccessControlService {
this._transactionSupportCache = supportsTransactions;
let localSession = session;
let shouldEndSession = false;
+ const rawResourceId = resourceId.toString();
+ const lock = {
+ key: `acl:${String(resourceType).toLowerCase()}:${
+ Types.ObjectId.isValid(rawResourceId)
+ ? new Types.ObjectId(rawResourceId).toString()
+ : rawResourceId.toLowerCase()
+ }`,
+ ownerId: new Types.ObjectId().toString(),
+ };
+ let lockHeld = false;
+ /** Keeps the lease alive through slow audit writes between fences. */
+ let heartbeat: NodeJS.Timeout | undefined;
try {
if (!Array.isArray(updatedPrincipals)) {
@@ -439,16 +529,87 @@ export class AccessControlService {
throw new Error(`Invalid resource ID: ${resourceId}`);
}
+ if (!supportsTransactions || (localSession && !localSession.inTransaction())) {
+ /**
+ * Without a transaction nothing else orders the audited read and the write, so a
+ * per-resource lease serializes concurrent updates of the same resource.
+ */
+ const deadline = Date.now() + leaseWaitMs;
+ const tryAcquire = () =>
+ this._dbMethods
+ .acquireResourceLock({
+ ...lock,
+ ttlMs: leaseTtlMs,
+ readyWaitMs: Math.max(0, deadline - Date.now()),
+ })
+ .catch((error) => {
+ logger.error('[AccessControlService.bulkUpdateResourcePermissions] Lock failed', {
+ ...getSafeErrorMetadata(error),
+ });
+ throw new ResourcePermissionsUnavailableError();
+ });
+ while (!(lockHeld = await tryAcquire())) {
+ if (Date.now() >= deadline) {
+ throw new ResourcePermissionsBusyError();
+ }
+ await new Promise((resolve) => setTimeout(resolve, RESOURCE_LOCK_POLL_MS));
+ }
+ }
+ if (lockHeld) {
+ heartbeat = setInterval(
+ () =>
+ void this._dbMethods
+ .renewResourceLock({ ...lock, ttlMs: leaseTtlMs })
+ .catch(() => undefined),
+ leaseTtlMs / 3,
+ );
+ }
if (!localSession && supportsTransactions) {
localSession = await this._mongoose.startSession();
localSession.startTransaction();
shouldEndSession = true;
}
const sessionOptions = localSession ? { session: localSession } : {};
+ const queryPrincipalId = (principal: BulkPrincipal) => {
+ if (principal.type === PrincipalType.ROLE) {
+ if (typeof principal.id !== 'string' || principal.id.trim().length === 0) {
+ throw new Error(`Invalid role ID: ${principal.id}`);
+ }
+ return principal.id;
+ }
+ const id = principal.id?.toString();
+ if (!id || !Types.ObjectId.isValid(id)) {
+ throw new Error(`Invalid principal ID: ${principal.id}`);
+ }
+ return new Types.ObjectId(id);
+ };
+ const affectedPrincipals: Array<{
+ principalType: string;
+ principalId?: string | Types.ObjectId;
+ }> = [];
+ for (const principal of [...updatedPrincipals, ...revokedPrincipals]) {
+ if (!principal?.type) continue;
+ try {
+ affectedPrincipals.push({
+ principalType: principal.type,
+ ...(principal.type !== PrincipalType.PUBLIC && {
+ principalId: queryPrincipalId(principal),
+ }),
+ });
+ } catch {
+ /** An invalid id is reported per principal below. */
+ }
+ }
const [roles, currentEntries] = await Promise.all([
this._dbMethods.findRolesByResourceType(resourceType),
- resourceType === ResourceType.AGENT
- ? this._dbMethods.findEntriesByResource(resourceType, resourceId, localSession)
+ affectedPrincipals.length > 0
+ ? this._dbMethods.findEntriesByPrincipalsAndResource(
+ affectedPrincipals,
+ resourceType,
+ resourceId,
+ localSession,
+ true,
+ )
: Promise.resolve([]),
]);
const rolesMap = new Map(roles.map((role) => [role.accessRoleId, role]));
@@ -457,24 +618,31 @@ export class AccessControlService {
updated: [],
revoked: [],
insightsChanges: [],
+ aclChanges: [],
errors: [],
};
const bulkWrites: Parameters<AllMethods['bulkWriteAclEntries']>[0] = [];
/** Role-only edits go through the atomic guarded write, not this batch. */
const roleBitsWrites: Parameters<AllMethods['replaceRoleBits']>[0] = [];
- const insightsChangesByBulkWriteIndex = new Map<number, InsightsPermissionChange>();
const updatedPrincipalKey = (principal: BulkPrincipal | null | undefined) => {
if (!principal?.type) return null;
- if (principal.type === PrincipalType.PUBLIC) return `${PrincipalType.PUBLIC}:null`;
- return principal.id == null ? null : `${principal.type}:${principal.id}`;
+ if (principal.type === PrincipalType.PUBLIC) return principalIdentity(principal.type, null);
+ return principal.id == null ? null : principalIdentity(principal.type, principal.id);
};
const lastUpdateIndexByPrincipal = new Map<string, number>();
updatedPrincipals.forEach((principal, index) => {
const key = updatedPrincipalKey(principal);
if (key != null) lastUpdateIndexByPrincipal.set(key, index);
});
+ /** An explicit public disable wins over a public update in the same request. */
+ const revokesPublic = revokedPrincipals.some(
+ (principal) => principal?.type === PrincipalType.PUBLIC,
+ );
const effectiveUpdatedPrincipals = updatedPrincipals.filter((principal, index) => {
+ if (revokesPublic && principal?.type === PrincipalType.PUBLIC) {
+ return false;
+ }
const key = updatedPrincipalKey(principal);
return key == null || lastUpdateIndexByPrincipal.get(key) === index;
});
@@ -485,31 +653,23 @@ export class AccessControlService {
* the permission just written. Public access is excluded because an explicit disable must win.
*/
const grantedPrincipalKeys = new Set<string>();
- const principalKey = (principal: BulkPrincipal) => `${principal.type}:${principal.id}`;
- const queryPrincipalId = (principal: BulkPrincipal) => {
- if (principal.type === PrincipalType.ROLE) {
- if (typeof principal.id !== 'string' || principal.id.trim().length === 0) {
- throw new Error(`Invalid role ID: ${principal.id}`);
- }
- return principal.id;
- }
- const id = principal.id?.toString();
- if (!id || !Types.ObjectId.isValid(id)) {
- throw new Error(`Invalid principal ID: ${principal.id}`);
- }
- return new Types.ObjectId(id);
- };
- const currentEntriesByPrincipal = new Map<string, IAclEntry>();
+ const principalKey = (principal: BulkPrincipal) =>
+ principalIdentity(principal.type, principal.id);
+ const currentEntriesByPrincipal = new Map<string, Pick<IAclEntry, 'permBits' | 'roleId'>>();
for (const entry of currentEntries) {
- const key = `${entry.principalType}:${entry.principalId == null ? 'null' : entry.principalId.toString()}`;
+ const key = principalIdentity(entry.principalType, entry.principalId);
const current = currentEntriesByPrincipal.get(key);
const entryHasInsights =
(entry.permBits & PermissionBits.VIEW_INSIGHTS) === PermissionBits.VIEW_INSIGHTS;
const currentHasInsights =
current != null &&
(current.permBits & PermissionBits.VIEW_INSIGHTS) === PermissionBits.VIEW_INSIGHTS;
+ /** Duplicate rows are audited as their combined permissions. */
+ const combinedBits = (current?.permBits ?? 0) | entry.permBits;
if (!current || (entryHasInsights && !currentHasInsights)) {
- currentEntriesByPrincipal.set(key, entry);
+ currentEntriesByPrincipal.set(key, { roleId: entry.roleId, permBits: combinedBits });
+ } else {
+ currentEntriesByPrincipal.set(key, { roleId: current.roleId, permBits: combinedBits });
}
}
@@ -541,11 +701,12 @@ export class AccessControlService {
}
const existingEntry = currentEntriesByPrincipal.get(
- `${principal.type}:${principal.type === PrincipalType.PUBLIC ? 'null' : principal.id}`,
+ principalIdentity(principal.type, principal.id),
);
const hadInsights =
+ resourceType === ResourceType.AGENT &&
((existingEntry?.permBits ?? 0) & PermissionBits.VIEW_INSIGHTS) ===
- PermissionBits.VIEW_INSIGHTS;
+ PermissionBits.VIEW_INSIGHTS;
const roleCanView = (role.permBits & PermissionBits.VIEW) === PermissionBits.VIEW;
const requestedInsights =
typeof principal.viewInsights === 'boolean' ? principal.viewInsights : undefined;
@@ -585,7 +746,10 @@ export class AccessControlService {
}),
},
};
- const bulkWriteIndex = bulkWrites.length;
+ /** `replaceRoleBits` swaps only the role mask and keeps every other stored bit. */
+ const writtenBits = preserveInsights
+ ? ((existingEntry?.permBits ?? 0) & ~ROLE_MASK) | (role.permBits & ROLE_MASK)
+ : permBits;
if (preserveInsights) {
roleBitsWrites.push({
filter: query,
@@ -612,21 +776,18 @@ export class AccessControlService {
memberIds: principal.memberIds,
...(resourceType === ResourceType.AGENT ? { viewInsights: wantsInsights } : {}),
});
+ const aclChange: PermissionChange = {
+ action: 'assigned',
+ principal: { type: principal.type, id: principal.id?.toString() ?? null },
+ previous: snapshotOf(existingEntry),
+ written: { permBits: writtenBits, roleId: role._id.toString() },
+ };
+ results.aclChanges.push(aclChange);
if (hadInsights !== wantsInsights) {
- const writtenEntry: InsightsWrittenEntry = {
- permBits,
- roleId: role._id,
- grantedBy,
- grantedAt,
- };
- const change: InsightsPermissionChange = {
+ results.insightsChanges.push({
+ ...aclChange,
action: wantsInsights ? 'assigned' : 'removed',
- previousEntry: existingEntry ?? null,
- writtenEntry,
- principal: { type: principal.type, id: principal.id, name: principal.name },
- };
- results.insightsChanges.push(change);
- insightsChangesByBulkWriteIndex.set(bulkWriteIndex, change);
+ });
}
if (principal.type !== PrincipalType.PUBLIC) {
grantedPrincipalKeys.add(principalKey(principal));
@@ -639,27 +800,20 @@ export class AccessControlService {
}
}
- if (roleBitsWrites.length > 0) {
- await this._dbMethods.replaceRoleBits(roleBitsWrites, {
- ...sessionOptions,
- maxAttempts: maxWriteAttempts,
- });
- }
-
- if (bulkWrites.length > 0) {
- const bulkWriteResult = await this._dbMethods.bulkWriteAclEntries(
- bulkWrites,
- sessionOptions,
- );
- for (const [writeIndex, change] of insightsChangesByBulkWriteIndex) {
- if (bulkWriteResult.upsertedIds?.[writeIndex] && change.previousEntry) {
- change.previousEntry = null;
- }
- }
- }
-
+ const revokeQueries: Array<{
+ principal: BulkPrincipal;
+ query: Record<string, unknown> | null;
+ }> = [];
+ const plannedRevokes = new Set<string>();
for (const principal of revokedPrincipals) {
try {
+ if (principal?.type) {
+ const identity = principalIdentity(principal.type, principal.id);
+ if (plannedRevokes.has(identity)) {
+ continue;
+ }
+ plannedRevokes.add(identity);
+ }
if (
principal?.type !== PrincipalType.PUBLIC &&
principal != null &&
@@ -675,33 +829,27 @@ export class AccessControlService {
if (principal.type !== PrincipalType.PUBLIC) {
query.principalId = queryPrincipalId(principal);
}
- const deleteResult = await this._dbMethods.deleteAclEntries(query, sessionOptions);
const existingEntry = currentEntriesByPrincipal.get(
- `${principal.type}:${principal.type === PrincipalType.PUBLIC ? 'null' : principal.id}`,
+ principalIdentity(principal.type, principal.id),
);
- results.revoked.push({
- type: principal.type,
- id: principal.id,
- name: principal.name,
- email: principal.email,
- source: principal.source,
- avatar: principal.avatar,
- description: principal.description,
- idOnTheSource: principal.idOnTheSource,
- memberCount: principal.memberCount,
- });
+ if (!existingEntry) {
+ /** Nothing was audited for it; deleting a row that appeared since the read would be unaudited. */
+ revokeQueries.push({ principal, query: null });
+ continue;
+ }
+ revokeQueries.push({ principal, query });
+ const change: PermissionChange = {
+ action: 'removed',
+ principal: { type: principal.type, id: principal.id?.toString() ?? null },
+ previous: snapshotOf(existingEntry),
+ written: null,
+ };
+ results.aclChanges.push(change);
if (
resourceType === ResourceType.AGENT &&
- deleteResult.deletedCount > 0 &&
- existingEntry &&
(existingEntry.permBits & PermissionBits.VIEW_INSIGHTS) === PermissionBits.VIEW_INSIGHTS
) {
- results.insightsChanges.push({
- action: 'removed',
- previousEntry: existingEntry,
- writtenEntry: null,
- principal: { type: principal.type, id: principal.id, name: principal.name },
- });
+ results.insightsChanges.push(change);
}
} catch (error) {
results.errors.push({
@@ -711,9 +859,73 @@ export class AccessControlService {
}
}
+ if (beforeWrite && (results.aclChanges.length > 0 || results.insightsChanges.length > 0)) {
+ await beforeWrite({
+ aclChanges: results.aclChanges,
+ insightsChanges: results.insightsChanges,
+ });
+ }
+
+ /** Re-validates and extends the lease before each write; a lost lease writes nothing more. */
+ const fence = async () => {
+ if (!lockHeld) {
+ return;
+ }
+ const renewed = await this._dbMethods
+ .renewResourceLock({ ...lock, ttlMs: leaseTtlMs })
+ .catch((error) => {
+ logger.error('[AccessControlService.bulkUpdateResourcePermissions] Renewal failed', {
+ ...getSafeErrorMetadata(error),
+ });
+ throw new ResourcePermissionsUnavailableError();
+ });
+ if (!renewed) {
+ throw new ResourcePermissionsBusyError();
+ }
+ };
+
+ await fence();
+ if (roleBitsWrites.length > 0) {
+ await this._dbMethods.replaceRoleBits(roleBitsWrites, {
+ ...sessionOptions,
+ maxAttempts: maxWriteAttempts,
+ });
+ }
+
+ if (bulkWrites.length > 0) {
+ await fence();
+ await this._dbMethods.bulkWriteAclEntries(bulkWrites, sessionOptions);
+ }
+
+ for (const { principal, query } of revokeQueries) {
+ if (query) {
+ await fence();
+ await this._dbMethods.deleteAclEntries(query, sessionOptions);
+ }
+ results.revoked.push({
+ type: principal.type,
+ id: principal.id,
+ name: principal.name,
+ email: principal.email,
+ source: principal.source,
+ avatar: principal.avatar,
+ description: principal.description,
+ idOnTheSource: principal.idOnTheSource,
+ memberCount: principal.memberCount,
+ });
+ }
+
if (shouldEndSession && supportsTransactions) {
await localSession?.commitTransaction();
}
+ /** A caller-owned transaction may still abort, so only a durable write is finalized. */
+ if (!localSession?.inTransaction()) {
+ await afterWrite?.().catch((error) =>
+ logger.error('[AccessControlService.bulkUpdateResourcePermissions] afterWrite failed', {
+ ...getSafeErrorMetadata(error),
+ }),
+ );
+ }
if (resourceType === ResourceType.PROMPTGROUP) {
await runAfterTransaction(localSession, () =>
this._dbMethods.invalidatePromptGroupAccessContext(),
@@ -739,88 +951,14 @@ export class AccessControlService {
if (shouldEndSession && localSession) {
await localSession.endSession();
}
- }
- }
-
- public async restoreInsightsPermissionChanges({
- resourceType,
- resourceId,
- changes = [],
- }: {
- resourceType: ResourceType;
- resourceId: string | Types.ObjectId;
- changes?: InsightsPermissionChange[];
- }): Promise<void> {
- const operations: Parameters<AllMethods['bulkWriteAclEntries']>[0] = changes.flatMap(
- (change): Parameters<AllMethods['bulkWriteAclEntries']>[0] => {
- const filter: Record<string, unknown> = {
- principalType: change.principal.type,
- resourceType,
- resourceId,
- };
- if (change.principal.type !== PrincipalType.PUBLIC) {
- if (change.principal.type === PrincipalType.ROLE) {
- filter.principalId = change.principal.id;
- } else {
- const id = change.principal.id?.toString();
- if (!id || !Types.ObjectId.isValid(id)) {
- throw new Error(`Invalid principal ID: ${change.principal.id}`);
- }
- filter.principalId = new Types.ObjectId(id);
- }
- }
- if (change.previousEntry && change.writtenEntry) {
- return [
- {
- replaceOne: {
- filter: { _id: change.previousEntry._id, ...change.writtenEntry },
- replacement: change.previousEntry,
- timestamps: false,
- },
- },
- {
- updateMany: {
- filter: {
- ...filter,
- ...change.writtenEntry,
- _id: { $ne: change.previousEntry._id },
- },
- update: {
- $set: {
- permBits: change.previousEntry.permBits,
- roleId: change.previousEntry.roleId,
- grantedBy: change.previousEntry.grantedBy,
- grantedAt: change.previousEntry.grantedAt,
- },
- },
- timestamps: false,
- },
- },
- ];
- }
- if (change.previousEntry) {
- return [
- {
- updateOne: {
- filter,
- update: { $setOnInsert: change.previousEntry },
- upsert: true,
- timestamps: false,
- },
- },
- ];
- }
- return [
- {
- deleteMany: {
- filter: { ...filter, ...change.writtenEntry },
- },
- },
- ];
- },
- );
- if (operations.length > 0) {
- await this._dbMethods.bulkWriteAclEntries(operations);
+ clearInterval(heartbeat);
+ if (lockHeld) {
+ await this._dbMethods.releaseResourceLock(lock).catch((error) =>
+ logger.error('[AccessControlService.bulkUpdateResourcePermissions] Lock release failed', {
+ ...getSafeErrorMetadata(error),
+ }),
+ );
+ }
}
}
diff --git a/packages/api/src/acl/audit.spec.ts b/packages/api/src/acl/audit.spec.ts
new file mode 100644
index 0000000000..1f3e0def78
--- /dev/null
+++ b/packages/api/src/acl/audit.spec.ts
@@ -0,0 +1,362 @@
+import { PrincipalType, ResourceType } from 'librechat-data-provider';
+import type { PermissionChange } from './types';
+import type { ServerRequest } from '~/types';
+import { runAuditedPermissionUpdate } from './audit';
+
+const req = {
+ user: { _id: { toString: () => 'actor-id' }, id: 'actor-id', name: 'Actor', tenantId: 't1' },
+ headers: {},
+} as unknown as ServerRequest;
+
+const grantChange = (id: string, permBits = 3): PermissionChange => ({
+ action: 'assigned',
+ previous: null,
+ written: { permBits, roleId: 'r1' },
+ principal: { type: PrincipalType.USER, id },
+});
+
+const revokeChange = (type: PrincipalType, id: string | null): PermissionChange => ({
+ action: 'removed',
+ previous: { permBits: 1, roleId: 'r1' },
+ written: null,
+ principal: { type, id },
+});
+
+function setup(recordImpl?: jest.Mock) {
+ const recordAuditEntry = recordImpl ?? jest.fn().mockResolvedValue(null);
+ const logger = { error: jest.fn() };
+ const write = jest.fn().mockResolvedValue('written');
+ const run = (
+ aclChanges: PermissionChange[],
+ failClosed: boolean,
+ options: {
+ insightsChanges?: PermissionChange[];
+ getAgent?: jest.Mock;
+ resourceType?: ResourceType;
+ r?: ServerRequest;
+ resourceId?: string;
+ } = {},
+ ) =>
+ runAuditedPermissionUpdate({
+ req: options.r ?? req,
+ resourceType: options.resourceType ?? ResourceType.PROMPTGROUP,
+ resourceId: options.resourceId ?? 'res-1',
+ failClosed,
+ deps: { recordAuditEntry, logger, getAgent: options.getAgent },
+ update: async (beforeWrite) => {
+ await beforeWrite({ aclChanges, insightsChanges: options.insightsChanges ?? [] });
+ return write();
+ },
+ });
+ return { recordAuditEntry, logger, write, run };
+}
+
+describe('runAuditedPermissionUpdate', () => {
+ test('targets an uppercase ObjectId hex by its lowercase form', async () => {
+ const { recordAuditEntry, run } = setup();
+ const hex = 'ABCDEF0123456789ABCDEF01';
+
+ await run([grantChange('u1')], true, { resourceId: hex });
+
+ expect(recordAuditEntry.mock.calls[0][0].target.id).toBe(hex.toLowerCase());
+ });
+
+ test('records grant, revoke and public entries with ids and bits only, before the write', async () => {
+ const { recordAuditEntry, write, run } = setup();
+ recordAuditEntry.mockImplementation(async () => {
+ expect(write).not.toHaveBeenCalled();
+ });
+
+ await expect(
+ run(
+ [
+ grantChange('u1'),
+ revokeChange(PrincipalType.GROUP, 'g1'),
+ revokeChange(PrincipalType.PUBLIC, null),
+ ],
+ true,
+ ),
+ ).resolves.toBe('written');
+
+ const inputs = recordAuditEntry.mock.calls.map(([input]) => input);
+ expect(inputs.map((i) => i.action)).toEqual([
+ 'acl.granted',
+ 'acl.revoked',
+ 'acl.public_changed',
+ ]);
+ expect(inputs[0]).toEqual(
+ expect.objectContaining({
+ outcome: 'success',
+ target: { type: ResourceType.PROMPTGROUP, id: 'res-1' },
+ tenantId: 't1',
+ metadata: {
+ principalType: 'user',
+ principalId: 'u1',
+ permBits: 3,
+ previousPermBits: 0,
+ roleId: 'r1',
+ previousRoleId: '',
+ resourceId: 'res-1',
+ },
+ }),
+ );
+ expect(inputs[2].metadata).toEqual(expect.objectContaining({ enabled: false }));
+ expect(recordAuditEntry.mock.calls[0][1]).toEqual({ failClosed: true });
+ });
+
+ test('skips a grant that changes nothing', async () => {
+ const { recordAuditEntry, write, run } = setup();
+ await run([{ ...grantChange('u1'), previous: { permBits: 3, roleId: 'r1' } }], true);
+ expect(recordAuditEntry).not.toHaveBeenCalled();
+ expect(write).toHaveBeenCalledTimes(1);
+ });
+
+ test('fail-closed: a failed audit write rejects the request and nothing is written', async () => {
+ const record = jest.fn().mockResolvedValueOnce(null).mockRejectedValueOnce(new Error('down'));
+ const { write, run } = setup(record);
+
+ await expect(run([grantChange('u1'), grantChange('u2')], true)).rejects.toMatchObject({
+ statusCode: 500,
+ });
+ expect(write).not.toHaveBeenCalled();
+ /** The entry already recorded is withdrawn with a matching failure entry. */
+ expect(record.mock.calls.map(([input]) => input.outcome)).toEqual([
+ 'success',
+ 'success',
+ 'failure',
+ ]);
+ });
+
+ test('fail-closed: no authenticated actor rejects before any write', async () => {
+ const { recordAuditEntry, write, run } = setup();
+
+ await expect(
+ run([grantChange('u1')], true, { r: { headers: {} } as unknown as ServerRequest }),
+ ).rejects.toThrow();
+ expect(recordAuditEntry).not.toHaveBeenCalled();
+ expect(write).not.toHaveBeenCalled();
+ });
+
+ test('opt-out: logs a failed audit write and applies the change anyway', async () => {
+ const record = jest.fn().mockRejectedValue(new Error('down'));
+ const { write, logger, run } = setup(record);
+
+ await expect(run([grantChange('u1'), grantChange('u2')], false)).resolves.toBe('written');
+ expect(record).toHaveBeenCalledTimes(2);
+ expect(write).toHaveBeenCalledTimes(1);
+ expect(logger.error).toHaveBeenCalledTimes(2);
+ });
+
+ test('an audit outage blocks a widening change but not a revocation', async () => {
+ const down = jest.fn().mockRejectedValue(new Error('down'));
+ const widen = setup(down);
+ await expect(widen.run([grantChange('u1')], true)).rejects.toMatchObject({ statusCode: 500 });
+ expect(widen.write).not.toHaveBeenCalled();
+
+ const record = jest.fn().mockRejectedValue(new Error('down'));
+ const narrow = setup(record);
+ await expect(
+ narrow.run(
+ [revokeChange(PrincipalType.USER, 'u1'), revokeChange(PrincipalType.PUBLIC, null)],
+ true,
+ ),
+ ).resolves.toBe('written');
+ expect(narrow.write).toHaveBeenCalledTimes(1);
+ /** Each entry is attempted before the write and once more after it. */
+ expect(record).toHaveBeenCalledTimes(4);
+ expect(narrow.logger.error).toHaveBeenCalled();
+ });
+
+ test('retries deferred revocation entries before the update returns, inside its lease', async () => {
+ const record = jest.fn();
+ const order: string[] = [];
+ record.mockImplementation(async (input) => {
+ order.push(`record:${input.action}`);
+ if (order.length === 1) throw new Error('down');
+ });
+ const { logger } = setup(record);
+ await runAuditedPermissionUpdate({
+ req,
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId: 'res-1',
+ failClosed: true,
+ deps: { recordAuditEntry: record, logger },
+ update: async (beforeWrite, afterWrite) => {
+ await beforeWrite({
+ aclChanges: [revokeChange(PrincipalType.USER, 'u1')],
+ insightsChanges: [],
+ });
+ order.push('write');
+ await afterWrite();
+ order.push('lease-released');
+ return 'written';
+ },
+ });
+ expect(order).toEqual(['record:acl.revoked', 'write', 'record:acl.revoked', 'lease-released']);
+ });
+
+ test('reports a pure bit reduction as a revocation', async () => {
+ const { recordAuditEntry, run } = setup();
+ const lowered = { ...grantChange('u1', 1), previous: { permBits: 3, roleId: 'r2' } };
+ await run([lowered], true);
+ expect(recordAuditEntry.mock.calls[0][0].action).toBe('acl.revoked');
+ });
+
+ test('an audit outage on a widening change carries a stable code', async () => {
+ const { run } = setup(jest.fn().mockRejectedValue(new Error('down')));
+ await expect(run([grantChange('u1')], true)).rejects.toMatchObject({
+ code: 'PERMISSION_AUDIT_UNAVAILABLE',
+ });
+ });
+
+ test('keeps success entries when the update fails after the write committed', async () => {
+ const { recordAuditEntry } = setup();
+ await expect(
+ runAuditedPermissionUpdate({
+ req,
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId: 'res-1',
+ failClosed: true,
+ deps: { recordAuditEntry, logger: { error: jest.fn() } },
+ update: async (beforeWrite, afterWrite) => {
+ await beforeWrite({ aclChanges: [grantChange('u1')], insightsChanges: [] });
+ await afterWrite();
+ throw new Error('cache down');
+ },
+ }),
+ ).rejects.toThrow('cache down');
+ expect(recordAuditEntry.mock.calls.map(([i]) => i.outcome)).toEqual(['success']);
+ });
+
+ test('every entry carries the database resource id, with or without the agent lookup', async () => {
+ const { recordAuditEntry, run } = setup();
+ await run([grantChange('u1')], true, {
+ resourceType: ResourceType.AGENT,
+ getAgent: jest.fn().mockRejectedValue(new Error('down')),
+ insightsChanges: [grantChange('u1')],
+ });
+ const metas = recordAuditEntry.mock.calls.map(([i]) => i.metadata.resourceId);
+ expect(metas.length).toBeGreaterThanOrEqual(2);
+ expect(new Set(metas)).toEqual(new Set(['res-1']));
+ });
+
+ test('audits principals and role ids in canonical form', async () => {
+ const { recordAuditEntry, run } = setup();
+ const hex = 'ABCDEF0123456789ABCDEF01';
+ await run([revokeChange(PrincipalType.USER, hex)], true);
+ expect(recordAuditEntry.mock.calls[0][0].metadata).toEqual(
+ expect.objectContaining({ principalId: hex.toLowerCase(), previousRoleId: 'r1', roleId: '' }),
+ );
+ });
+
+ test('a reduced permission set is a narrowing change, an added bit is a widening one', async () => {
+ const record = jest.fn().mockRejectedValue(new Error('down'));
+ const { write, run } = setup(record);
+ const lowered = { ...grantChange('u1', 1), previous: { permBits: 3, roleId: 'r2' } };
+ await expect(run([lowered], true)).resolves.toBe('written');
+ expect(write).toHaveBeenCalledTimes(1);
+
+ const raised = { ...grantChange('u1', 3), previous: { permBits: 1, roleId: 'r2' } };
+ const blocked = setup(record);
+ await expect(blocked.run([raised], true)).rejects.toMatchObject({ statusCode: 500 });
+ expect(blocked.write).not.toHaveBeenCalled();
+ });
+
+ test('a failed ACL write appends failure entries for the recorded changes and rethrows', async () => {
+ const { recordAuditEntry } = setup();
+ const writeError = new Error('acl write down');
+
+ await expect(
+ runAuditedPermissionUpdate({
+ req,
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId: 'res-1',
+ failClosed: true,
+ deps: { recordAuditEntry, logger: { error: jest.fn() } },
+ update: async (beforeWrite) => {
+ await beforeWrite({
+ aclChanges: [grantChange('u1'), revokeChange(PrincipalType.GROUP, 'g1')],
+ insightsChanges: [],
+ });
+ throw writeError;
+ },
+ }),
+ ).rejects.toBe(writeError);
+
+ expect(recordAuditEntry.mock.calls.map(([input]) => [input.action, input.outcome])).toEqual([
+ ['acl.granted', 'success'],
+ ['acl.revoked', 'success'],
+ ['acl.granted', 'failure'],
+ ['acl.revoked', 'failure'],
+ ]);
+ });
+
+ test('a failure entry that cannot be persisted does not mask the write error', async () => {
+ const record = jest
+ .fn()
+ .mockResolvedValueOnce(null)
+ .mockRejectedValueOnce(new Error('audit down'));
+ const logger = { error: jest.fn() };
+ const writeError = new Error('acl write down');
+
+ await expect(
+ runAuditedPermissionUpdate({
+ req,
+ resourceType: ResourceType.PROMPTGROUP,
+ resourceId: 'res-1',
+ failClosed: true,
+ deps: { recordAuditEntry: record, logger },
+ update: async (beforeWrite) => {
+ await beforeWrite({ aclChanges: [grantChange('u1')], insightsChanges: [] });
+ throw writeError;
+ },
+ }),
+ ).rejects.toBe(writeError);
+ expect(logger.error).toHaveBeenCalled();
+ });
+
+ test('audits ACL and Insights changes together, ACL entries first', async () => {
+ const { recordAuditEntry, write, run } = setup();
+ const getAgent = jest.fn().mockResolvedValue({ id: 'agent-logical', name: 'Agent' });
+
+ await run([grantChange('u1')], true, {
+ resourceType: ResourceType.AGENT,
+ insightsChanges: [grantChange('u1')],
+ getAgent,
+ });
+
+ expect(recordAuditEntry.mock.calls.map(([input]) => input.action)).toEqual([
+ 'acl.granted',
+ 'permission.insights_assigned',
+ ]);
+ /** Both event families name the agent by the same id. */
+ expect(recordAuditEntry.mock.calls.map(([input]) => input.target.id)).toEqual([
+ 'agent-logical',
+ 'agent-logical',
+ ]);
+ expect(getAgent).toHaveBeenCalledTimes(1);
+ expect(write).toHaveBeenCalledTimes(1);
+ });
+
+ test.each([true, false])(
+ 'still audits Insights with the resource id when the agent lookup fails (failClosed=%p)',
+ async (failClosed) => {
+ const { recordAuditEntry, logger, run } = setup();
+
+ await run([], failClosed, {
+ resourceType: ResourceType.AGENT,
+ insightsChanges: [grantChange('u1')],
+ getAgent: jest.fn().mockRejectedValue(new Error('lookup down')),
+ });
+
+ expect(recordAuditEntry).toHaveBeenCalledWith(
+ expect.objectContaining({
+ action: 'permission.insights_assigned',
+ target: { type: ResourceType.AGENT, id: 'res-1', name: 'res-1' },
+ }),
+ { failClosed },
+ );
+ expect(logger.error).toHaveBeenCalled();
+ },
+ );
+});
diff --git a/packages/api/src/acl/audit.ts b/packages/api/src/acl/audit.ts
new file mode 100644
index 0000000000..f9a634b35c
--- /dev/null
+++ b/packages/api/src/acl/audit.ts
@@ -0,0 +1,344 @@
+import { PrincipalType, ResourceType } from 'librechat-data-provider';
+import type { RecordAuditEntryInput } from '@librechat/data-schemas';
+import type { RecordAuditEntry } from '~/admin/audit';
+import type { PermissionChange } from './types';
+import type { ServerRequest } from '~/types';
+import { getAuditActor, isAuditFailClosed } from '~/admin/audit';
+import { PERMISSION_AUDIT_UNAVAILABLE } from './codes';
+import { getSafeErrorMetadata } from '~/utils/errors';
+import { buildAuditContext } from '~/admin/context';
+
+const OBJECT_ID_HEX = /^[0-9a-f]{24}$/i;
+
+/** Mongo casts ObjectId hex of any case to one id, so non-role principals are audited in lowercase. */
+const canonicalPrincipalId = (type: string, id: unknown): string => {
+ const raw = id == null ? '' : id.toString();
+ return type !== PrincipalType.ROLE && OBJECT_ID_HEX.test(raw) ? raw.toLowerCase() : raw;
+};
+
+export type AuditPlanItem = {
+ input: RecordAuditEntryInput;
+ /** Grants or widens access; only these are held back when the audit store is down. */
+ widening: boolean;
+};
+
+type AclAuditDeps = {
+ recordAuditEntry: RecordAuditEntry;
+ logger: { error: (message: string, error?: unknown) => void };
+};
+
+const safeMeta = (error: unknown) => ({ ...getSafeErrorMetadata(error) });
+
+function statusError(message: string, code?: string): Error & { statusCode: number } {
+ return Object.assign(new Error(message), { statusCode: 500, ...(code && { code }) });
+}
+
+/** A new grant, an added permission bit or a public enable; anything else only narrows access. */
+const widensAccess = (change: PermissionChange): boolean =>
+ change.action === 'assigned' &&
+ (change.previous == null || ((change.written?.permBits ?? 0) & ~change.previous.permBits) !== 0);
+
+function buildEvent(
+ change: PermissionChange,
+): Pick<RecordAuditEntryInput, 'action' | 'metadata'> | null {
+ const { previous, written, principal } = change;
+ const isPublic = principal.type === PrincipalType.PUBLIC;
+ const previousBits = previous?.permBits ?? 0;
+ const permBits = written?.permBits ?? 0;
+ if (change.action === 'assigned') {
+ const unchanged =
+ previous != null &&
+ written != null &&
+ previous.roleId === written.roleId &&
+ previousBits === permBits;
+ if (unchanged) {
+ return null;
+ }
+ }
+ const metadata = {
+ principalType: principal.type,
+ principalId: canonicalPrincipalId(principal.type, principal.id),
+ permBits,
+ previousPermBits: previousBits,
+ roleId: written?.roleId ?? '',
+ previousRoleId: previous?.roleId ?? '',
+ };
+ if (isPublic) {
+ return {
+ action: 'acl.public_changed',
+ metadata: { ...metadata, enabled: change.action === 'assigned' },
+ };
+ }
+ /** A pure reduction of bits removes access, so it is reported as a revocation. */
+ const reduced =
+ previous != null && (previousBits & ~permBits) !== 0 && (permBits & ~previousBits) === 0;
+ return {
+ action: change.action === 'assigned' && !reduced ? 'acl.granted' : 'acl.revoked',
+ metadata,
+ };
+}
+
+type InsightsPlanDeps = {
+ getAgent: (
+ filter: Record<string, unknown>,
+ projection: string,
+ ) => Promise<{ id: string; name?: string } | null>;
+ logger: AclAuditDeps['logger'];
+};
+
+type AgentTarget = { id: string; name: string };
+
+/** The lookup only decorates the entry; a failure must not skip the audit. */
+async function lookupAgentTarget({
+ req,
+ resourceId,
+ deps,
+}: {
+ req: ServerRequest;
+ resourceId: string;
+ deps: InsightsPlanDeps;
+}): Promise<AgentTarget> {
+ let agent: { id: string; name?: string } | null = null;
+ try {
+ agent = await deps.getAgent(
+ {
+ _id: resourceId,
+ ...(req.user?.tenantId
+ ? { tenantId: req.user.tenantId }
+ : { tenantId: { $exists: false } }),
+ },
+ '_id id name',
+ );
+ } catch (error) {
+ deps.logger.error('[InsightsPermissions] Audit target lookup failed', safeMeta(error));
+ }
+ return { id: agent?.id ?? resourceId, name: agent?.name || agent?.id || resourceId };
+}
+
+/** Builds one `permission.insights_*` entry per Insights transition. */
+export async function planInsightsAudit(args: {
+ req: ServerRequest;
+ resourceId: string;
+ changes: PermissionChange[];
+ deps: InsightsPlanDeps;
+ /** The resolved agent target, when the caller already looked it up. */
+ target?: AgentTarget;
+}): Promise<AuditPlanItem[]> {
+ const { req, resourceId, changes, deps } = args;
+ if (changes.length === 0) {
+ return [];
+ }
+ const actor = getAuditActor(req);
+ if (!actor) {
+ throw statusError('Authenticated user required for Insights permission audit');
+ }
+ const target = args.target ?? (await lookupAgentTarget({ req, resourceId, deps }));
+ const context = buildAuditContext(req);
+ return changes.map((change) => ({
+ widening: change.action === 'assigned',
+ input: {
+ action:
+ change.action === 'assigned'
+ ? 'permission.insights_assigned'
+ : 'permission.insights_removed',
+ outcome: 'success',
+ severity: 'warning',
+ actor,
+ target: { type: 'agent', ...target },
+ metadata: {
+ principalType: change.principal.type,
+ principalId: canonicalPrincipalId(change.principal.type, change.principal.id),
+ /** The database id, present however the agent's logical id resolved, for correlation. */
+ resourceId,
+ },
+ context,
+ tenantId: req.user?.tenantId,
+ },
+ }));
+}
+
+/** Builds one `acl.*` entry per effective grant, revoke and public change. */
+export function planResourceAudit({
+ req,
+ resourceType,
+ resourceId,
+ changes,
+ targetId = resourceId,
+}: {
+ req: ServerRequest;
+ resourceType: ResourceType;
+ resourceId: string;
+ changes: PermissionChange[];
+ /** The id the resource is audited under; defaults to the route id. */
+ targetId?: string;
+}): AuditPlanItem[] {
+ if (changes.length === 0) {
+ return [];
+ }
+ const actor = getAuditActor(req);
+ if (!actor) {
+ throw statusError('Authenticated user required for permission audit');
+ }
+ const context = buildAuditContext(req);
+ const items: AuditPlanItem[] = [];
+ for (const change of changes) {
+ const event = buildEvent(change);
+ if (event) {
+ items.push({
+ widening: widensAccess(change),
+ input: {
+ ...event,
+ metadata: { ...event.metadata, resourceId },
+ outcome: 'success',
+ severity: 'warning',
+ actor,
+ target: { type: resourceType, id: targetId },
+ context,
+ tenantId: req.user?.tenantId,
+ },
+ });
+ }
+ }
+ return items;
+}
+
+export type PermissionPlan = {
+ aclChanges: PermissionChange[];
+ insightsChanges: PermissionChange[];
+};
+
+/**
+ * Audits a resource permission update before it is written. `update` receives a
+ * `beforeWrite` hook to call with the computed changes ahead of any ACL write:
+ * it records one `acl.*` entry per grant, revoke and public change, plus the
+ * agent Insights entries when `getAgent` is given. With `failClosed`, a failed
+ * audit write throws, so the request fails with the ACL untouched. If the ACL
+ * write then fails, the entries already recorded are followed by matching
+ * `failure` entries (best effort), so the log never claims a change that did
+ * not happen. Entries carry ids and permission bits only, never names or emails.
+ */
+export async function runAuditedPermissionUpdate<T>({
+ req,
+ resourceType,
+ resourceId: rawResourceId,
+ failClosed = isAuditFailClosed(),
+ deps,
+ update,
+}: {
+ req: ServerRequest;
+ resourceType: ResourceType;
+ resourceId: string;
+ /** Overrides `isAuditFailClosed()`. */
+ failClosed?: boolean;
+ deps: AclAuditDeps & { getAgent?: InsightsPlanDeps['getAgent'] };
+ /** `afterWrite` retries deferred revocation entries; call it while the resource lease is still held. */
+ update: (
+ beforeWrite: (plan: PermissionPlan) => Promise<void>,
+ afterWrite: () => Promise<void>,
+ ) => Promise<T>;
+}): Promise<T> {
+ /** Mongoose casts uppercase and lowercase ObjectId hex to one id, so audit targets use the lowercase form. */
+ const resourceId = OBJECT_ID_HEX.test(rawResourceId)
+ ? rawResourceId.toLowerCase()
+ : rawResourceId;
+ const recorded: AuditPlanItem[] = [];
+ const recordFailures = async () => {
+ const items = recorded.splice(0);
+ for (const item of items) {
+ try {
+ await deps.recordAuditEntry({ ...item.input, outcome: 'failure' });
+ } catch (error) {
+ deps.logger.error(
+ '[ResourcePermissions] Audit failure entry not persisted',
+ safeMeta(error),
+ );
+ }
+ }
+ };
+ const deferred: AuditPlanItem[] = [];
+ /** Set once the write is durable; a later error must not relabel recorded changes as failed. */
+ let committed = false;
+ const afterWrite = async () => {
+ committed = true;
+ await flushDeferred();
+ };
+ const flushDeferred = async () => {
+ for (const item of deferred.splice(0)) {
+ try {
+ await deps.recordAuditEntry(item.input);
+ } catch (error) {
+ deps.logger.error('[ResourcePermissions] Revocation audit not persisted', safeMeta(error));
+ }
+ }
+ };
+ try {
+ const result = await update(async ({ aclChanges, insightsChanges }) => {
+ let items: AuditPlanItem[];
+ try {
+ const insightsDeps =
+ resourceType === ResourceType.AGENT && deps.getAgent
+ ? { getAgent: deps.getAgent, logger: deps.logger }
+ : null;
+ const target =
+ insightsDeps && (aclChanges.length > 0 || insightsChanges.length > 0)
+ ? await lookupAgentTarget({ req, resourceId, deps: insightsDeps })
+ : undefined;
+ items = [
+ ...planResourceAudit({
+ req,
+ resourceType,
+ resourceId,
+ changes: aclChanges,
+ targetId: target?.id,
+ }),
+ ...(insightsDeps
+ ? await planInsightsAudit({
+ req,
+ resourceId,
+ changes: insightsChanges,
+ deps: insightsDeps,
+ target,
+ })
+ : []),
+ ];
+ } catch (error) {
+ if (failClosed) {
+ throw error;
+ }
+ deps.logger.error('[ResourcePermissions] Audit skipped', safeMeta(error));
+ return;
+ }
+ /** Widening entries go first so their failure aborts before anything is recorded. */
+ const ordered = [
+ ...items.filter((item) => item.widening),
+ ...items.filter((item) => !item.widening),
+ ];
+ for (const item of ordered) {
+ try {
+ await deps.recordAuditEntry(item.input, { failClosed });
+ recorded.push(item);
+ } catch (error) {
+ if (!failClosed) {
+ deps.logger.error('[ResourcePermissions] Audit persistence failed', safeMeta(error));
+ continue;
+ }
+ if (!item.widening) {
+ /** Removing access must not wait on the audit store; the entry is retried after the write. */
+ deps.logger.error('[ResourcePermissions] Revocation audit deferred', safeMeta(error));
+ deferred.push(item);
+ continue;
+ }
+ await recordFailures();
+ throw statusError('Permission change could not be audited', PERMISSION_AUDIT_UNAVAILABLE);
+ }
+ }
+ }, afterWrite);
+ await flushDeferred();
+ return result;
+ } catch (error) {
+ if (!committed) {
+ await recordFailures();
+ }
+ throw error;
+ }
+}
diff --git a/packages/api/src/acl/codes.ts b/packages/api/src/acl/codes.ts
new file mode 100644
index 0000000000..a0c7e04e5d
--- /dev/null
+++ b/packages/api/src/acl/codes.ts
@@ -0,0 +1,2 @@
+export const PERMISSION_AUDIT_UNAVAILABLE = 'PERMISSION_AUDIT_UNAVAILABLE';
+export const RESOURCE_PERMISSIONS_UNAVAILABLE = 'RESOURCE_PERMISSIONS_UNAVAILABLE';
diff --git a/packages/api/src/acl/insightsPermissions.spec.ts b/packages/api/src/acl/insightsPermissions.spec.ts
index cdbe4bdaaa..46001ce4c3 100644
--- a/packages/api/src/acl/insightsPermissions.spec.ts
+++ b/packages/api/src/acl/insightsPermissions.spec.ts
@@ -1,10 +1,5 @@
import { PermissionBits, PrincipalType, ResourceType, SystemRoles } from 'librechat-data-provider';
-import type { ServerRequest } from '~/types';
-import {
- auditInsightsPermissionChanges,
- maskAgentInsightsBit,
- validateInsightsPermissionUpdates,
-} from './insightsPermissions';
+import { maskAgentInsightsBit, validateInsightsPermissionUpdates } from './insightsPermissions';
describe('Insights permissions', () => {
test('allows only admins to assign a boolean agent Insights grant', () => {
@@ -33,41 +28,4 @@ describe('Insights permissions', () => {
}),
).toBe(PermissionBits.VIEW);
});
-
- test('restores unaudited changes when fail-closed audit persistence fails', async () => {
- const restore = jest.fn().mockResolvedValue(undefined);
- const record = jest.fn().mockRejectedValue(new Error('audit unavailable'));
- const change = {
- action: 'assigned' as const,
- previousEntry: null,
- writtenEntry: null,
- principal: { type: PrincipalType.ROLE, id: SystemRoles.USER },
- };
- const req = {
- user: {
- _id: { toString: () => 'admin-id' },
- id: 'admin-id',
- email: '[email protected]',
- role: SystemRoles.ADMIN,
- tenantId: 'tenant-a',
- },
- headers: {},
- } as unknown as ServerRequest;
-
- await expect(
- auditInsightsPermissionChanges({
- req,
- resourceId: 'resource-id',
- changes: [change],
- failClosed: true,
- deps: {
- getAgent: jest.fn().mockResolvedValue({ id: 'agent-id', name: 'Agent' }),
- recordAuditEntry: record,
- restoreInsightsPermissionChanges: restore,
- logger: { error: jest.fn() },
- },
- }),
- ).rejects.toMatchObject({ statusCode: 500 });
- expect(restore).toHaveBeenCalledWith([change]);
- });
});
diff --git a/packages/api/src/acl/insightsPermissions.ts b/packages/api/src/acl/insightsPermissions.ts
index 7f03c3a468..453051245a 100644
--- a/packages/api/src/acl/insightsPermissions.ts
+++ b/packages/api/src/acl/insightsPermissions.ts
@@ -1,52 +1,10 @@
import { PermissionBits, PrincipalType, ResourceType, SystemRoles } from 'librechat-data-provider';
-import type {
- IAclEntry,
- IUser,
- RecordAuditEntryInput,
- RecordAuditEntryOptions,
-} from '@librechat/data-schemas';
import type { TPrincipal } from 'librechat-data-provider';
-import type { Types } from 'mongoose';
-import type { ServerRequest } from '~/types';
-import { buildAuditContext } from '~/admin/context';
-export type InsightsPermissionPrincipal = {
- type: PrincipalType;
- id?: string | Types.ObjectId | null;
- name?: string;
-};
-
-export type InsightsWrittenEntry = {
- permBits: number;
- roleId: Types.ObjectId;
- grantedBy: string | Types.ObjectId;
- grantedAt: Date;
-};
-
-export type InsightsPermissionChange = {
- action: 'assigned' | 'removed';
- previousEntry: IAclEntry | null;
- writtenEntry: InsightsWrittenEntry | null;
- principal: InsightsPermissionPrincipal;
-};
+export type { PermissionChange, PermissionSnapshot } from './types';
type ValidationResult = { status: 400 | 403; error: string } | null;
-type AuditDeps = {
- getAgent: (
- filter: Record<string, unknown>,
- projection: string,
- ) => Promise<{ id: string; name?: string } | null>;
- recordAuditEntry: (
- input: RecordAuditEntryInput,
- options?: RecordAuditEntryOptions,
- ) => Promise<unknown>;
- restoreInsightsPermissionChanges: (changes: InsightsPermissionChange[]) => Promise<void>;
- logger: {
- error: (message: string, error?: unknown) => void;
- };
-};
-
const hasOwn = (value: unknown, key: string): boolean =>
value != null && typeof value === 'object' && Object.prototype.hasOwnProperty.call(value, key);
@@ -138,92 +96,3 @@ export function getInsightsPrincipalState({
: {}),
};
}
-
-function statusError(message: string): Error & { statusCode: number } {
- return Object.assign(new Error(message), { statusCode: 500 });
-}
-
-export async function auditInsightsPermissionChanges({
- req,
- resourceId,
- changes,
- failClosed,
- deps,
-}: {
- req: ServerRequest;
- resourceId: string;
- changes: InsightsPermissionChange[];
- failClosed: boolean;
- deps: AuditDeps;
-}): Promise<void> {
- if (changes.length === 0) {
- return;
- }
-
- const user = req.user as IUser | undefined;
- const actorId = user?._id?.toString() ?? user?.id;
- if (!user || !actorId) {
- throw statusError('Authenticated user required for Insights permission audit');
- }
-
- let agent: { id: string; name?: string } | null;
- try {
- agent = await deps.getAgent(
- {
- _id: resourceId,
- ...(user.tenantId ? { tenantId: user.tenantId } : { tenantId: { $exists: false } }),
- },
- '_id id name',
- );
- if (!agent) {
- throw new Error('Agent not found for Insights permission audit');
- }
- } catch (error) {
- if (!failClosed) {
- deps.logger.error('[InsightsPermissions] Audit target lookup failed', error);
- return;
- }
- await deps
- .restoreInsightsPermissionChanges(changes)
- .catch((restoreError) =>
- deps.logger.error('[InsightsPermissions] Permission rollback failed', restoreError),
- );
- throw statusError(
- error instanceof Error ? error.message : 'Insights audit target lookup failed',
- );
- }
-
- const actorName = user.name || user.username || user.email || actorId;
- for (let index = 0; index < changes.length; index++) {
- const change = changes[index];
- const input: RecordAuditEntryInput = {
- action:
- change.action === 'assigned'
- ? 'permission.insights_assigned'
- : 'permission.insights_removed',
- outcome: 'success',
- severity: 'warning',
- actor: { type: 'user', id: actorId, name: actorName },
- target: { type: ResourceType.AGENT, id: agent.id, name: agent.name || agent.id },
- metadata: {
- principalType: change.principal.type,
- principalId: change.principal.id?.toString() ?? '',
- },
- context: buildAuditContext(req),
- tenantId: user.tenantId,
- };
- try {
- await deps.recordAuditEntry(input, { failClosed });
- } catch (error) {
- if (failClosed) {
- await deps
- .restoreInsightsPermissionChanges(changes.slice(index))
- .catch((restoreError) =>
- deps.logger.error('[InsightsPermissions] Permission rollback failed', restoreError),
- );
- throw statusError(error instanceof Error ? error.message : 'Insights audit failed');
- }
- deps.logger.error('[InsightsPermissions] Audit persistence failed', error);
- }
- }
-}
diff --git a/packages/api/src/acl/rolePermissions.spec.ts b/packages/api/src/acl/rolePermissions.spec.ts
index 74fb297537..038f6fd742 100644
--- a/packages/api/src/acl/rolePermissions.spec.ts
+++ b/packages/api/src/acl/rolePermissions.spec.ts
@@ -219,39 +219,6 @@ describe('role-only ACL writes', () => {
expect((await entries.findOne(filter))!.permBits).toBe(19);
});
- test('keeps explicit Insights upsert indices correct after expanded role-only writes', async () => {
- await seed(17);
- raceBeforeWrite(() => entries.deleteMany(filter));
- const result = await service.bulkUpdateResourcePermissions({
- resourceType: ResourceType.AGENT,
- resourceId,
- updatedPrincipals: [
- {
- type: PrincipalType.USER,
- id: new Types.ObjectId().toString(),
- accessRoleId: AccessRoleIds.AGENT_EDITOR,
- },
- {
- type: PrincipalType.USER,
- id: userId.toString(),
- accessRoleId: AccessRoleIds.AGENT_EDITOR,
- viewInsights: false,
- },
- ],
- grantedBy,
- });
- expect(result.insightsChanges).toEqual([
- expect.objectContaining({ action: 'removed', previousEntry: null }),
- ]);
- await service.restoreInsightsPermissionChanges({
- resourceType: ResourceType.AGENT,
- resourceId,
- changes: result.insightsChanges,
- });
- expect(await entries.findOne(filter)).toBeNull();
- expect(await entries.countDocuments({ resourceId })).toBe(1);
- });
-
test('initializes missing legacy permission fields without inheriting Insights', async () => {
await seed(1);
await entries.collection.updateMany(filter, { $unset: { permBits: '' } });
diff --git a/packages/api/src/acl/types.ts b/packages/api/src/acl/types.ts
new file mode 100644
index 0000000000..4632cf052e
--- /dev/null
+++ b/packages/api/src/acl/types.ts
@@ -0,0 +1,14 @@
+import type { PrincipalType } from 'librechat-data-provider';
+
+export type PermissionSnapshot = {
+ permBits: number;
+ roleId: string;
+};
+
+/** A grant, update or revoke on one principal, as plain ids and bits. */
+export type PermissionChange = {
+ action: 'assigned' | 'removed';
+ principal: { type: PrincipalType; id?: string | null };
+ previous: PermissionSnapshot | null;
+ written: PermissionSnapshot | null;
+};
diff --git a/packages/api/src/admin/audit.spec.ts b/packages/api/src/admin/audit.spec.ts
index b565bf488b..a80e0c55e1 100644
--- a/packages/api/src/admin/audit.spec.ts
+++ b/packages/api/src/admin/audit.spec.ts
@@ -1,5 +1,5 @@
import type { RecordAuditEntryInput } from '@librechat/data-schemas';
-import { recordSystemAuditEntry } from './audit';
+import { isAuditFailClosed, recordSystemAuditEntry } from './audit';
const input = { action: 'role.synced' } as unknown as RecordAuditEntryInput;
@@ -27,3 +27,17 @@ describe('recordSystemAuditEntry', () => {
).resolves.toBeUndefined();
});
});
+
+describe('isAuditFailClosed', () => {
+ it.each([
+ [undefined, true],
+ ['', true],
+ ['true', true],
+ ['False', true],
+ ['0', true],
+ ['false', false],
+ ])('AUDIT_LOG_FAIL_CLOSED=%p -> %p', (value, expected) => {
+ const env = value === undefined ? {} : { AUDIT_LOG_FAIL_CLOSED: value };
+ expect(isAuditFailClosed(env as NodeJS.ProcessEnv)).toBe(expected);
+ });
+});
diff --git a/packages/api/src/admin/audit.ts b/packages/api/src/admin/audit.ts
index e3df2b7d68..f3e97096e2 100644
--- a/packages/api/src/admin/audit.ts
+++ b/packages/api/src/admin/audit.ts
@@ -19,6 +19,16 @@ export type AdminAuditEvent = Omit<RecordAuditEntryInput, 'actor' | 'context'>;
export type AdminAuditEmitter = (req: ServerRequest, event: AdminAuditEvent) => Promise<void>;
+/**
+ * Whether resource permission updates fail closed. Their audit entry is written
+ * before the ACL changes, so a failed write rejects the request with nothing
+ * applied. On unless the operator opts out with `AUDIT_LOG_FAIL_CLOSED=false`;
+ * any other value keeps the secure default so a typo cannot silently disable it.
+ */
+export function isAuditFailClosed(env: NodeJS.ProcessEnv = process.env): boolean {
+ return env.AUDIT_LOG_FAIL_CLOSED !== 'false';
+}
+
/** Resolves the authenticated caller as an audit actor; null when unauthenticated. */
export function getAuditActor(req: ServerRequest): AuditActorInput | null {
const user = req.user;
diff --git a/packages/api/src/admin/grants.spec.ts b/packages/api/src/admin/grants.spec.ts
index fd22a2c7f3..547a8562d8 100644
--- a/packages/api/src/admin/grants.spec.ts
+++ b/packages/api/src/admin/grants.spec.ts
@@ -1,6 +1,7 @@
+import { inspect } from 'util';
import { Types } from 'mongoose';
import { PrincipalType } from 'librechat-data-provider';
-import { SystemCapabilities, expandImplications } from '@librechat/data-schemas';
+import { SystemCapabilities, expandImplications, logger } from '@librechat/data-schemas';
import type { ISystemGrant } from '@librechat/data-schemas';
import type { Response } from 'express';
import type { ServerRequest } from '~/types/http';
@@ -1245,12 +1246,15 @@ describe('createAdminGrantsHandlers', () => {
expect(status).toHaveBeenCalledWith(201);
});
- it('does not audit re-assigning a capability the role already holds', async () => {
+ it('audits re-assigning a held capability, marked as a re-assert', async () => {
const recordAuditEntry = jest.fn().mockResolvedValue(undefined);
const deps = createDeps({
recordAuditEntry,
- // the upsert reports created:false → idempotent re-assert, no change
- grantCapability: jest.fn().mockResolvedValue({ grant: mockGrant(), created: false }),
+ auditFailClosed: true,
+ getCapabilitiesForPrincipal: jest
+ .fn()
+ .mockResolvedValue([mockGrant({ tenantId: 'tenant-1' })]),
+ grantCapability: jest.fn().mockResolvedValue({ grant: mockGrant(), created: true }),
});
const handlers = createAdminGrantsHandlers(deps);
const { req, res, status } = createReqRes({ body: assignBody, user: reqUser() });
@@ -1258,7 +1262,42 @@ describe('createAdminGrantsHandlers', () => {
await handlers.assignGrant(req, res);
expect(status).toHaveBeenCalledWith(201);
- expect(recordAuditEntry).not.toHaveBeenCalled();
+ expect(recordAuditEntry).toHaveBeenCalledTimes(1);
+ expect(recordAuditEntry.mock.calls[0][0]).toMatchObject({
+ action: 'grant.assigned',
+ metadata: { reassert: true },
+ });
+ });
+
+ it('logs a failed fail-open audit write without the raw error', async () => {
+ const recordAuditEntry = jest
+ .fn()
+ .mockRejectedValue(new Error('SELECT secret FROM audit WHERE token=abc'));
+ const deps = createDeps({ recordAuditEntry });
+ const handlers = createAdminGrantsHandlers(deps);
+ const { req, res } = createReqRes({ body: assignBody, user: reqUser() });
+ (logger.error as jest.Mock).mockClear();
+
+ await handlers.assignGrant(req, res);
+
+ const args = (logger.error as jest.Mock).mock.calls.flat();
+ expect(args.some((arg) => arg instanceof Error)).toBe(false);
+ expect(inspect(args, { depth: 5 })).not.toContain('SELECT secret');
+ });
+
+ it('fail-closed: an audit outage answers with the stable audit code', async () => {
+ const recordAuditEntry = jest.fn().mockRejectedValue(new Error('audit down'));
+ const deps = createDeps({ recordAuditEntry, auditFailClosed: true });
+ const handlers = createAdminGrantsHandlers(deps);
+ const { req, res, status, json } = createReqRes({ body: assignBody, user: reqUser() });
+
+ await handlers.assignGrant(req, res);
+
+ expect(status).toHaveBeenCalledWith(500);
+ expect(json).toHaveBeenCalledWith(
+ expect.objectContaining({ code: 'PERMISSION_AUDIT_UNAVAILABLE' }),
+ );
+ expect(deps.grantCapability).not.toHaveBeenCalled();
});
it('fail-open: a failed audit write does not fail the grant', async () => {
@@ -1272,7 +1311,7 @@ describe('createAdminGrantsHandlers', () => {
expect(status).toHaveBeenCalledWith(201);
});
- it('fail-closed: a failed assign audit rolls back the grant and 500s', async () => {
+ it('fail-closed: an audit outage blocks an assign and nothing is granted', async () => {
const recordAuditEntry = jest.fn().mockRejectedValue(new Error('audit down'));
const deps = createDeps({ recordAuditEntry, auditFailClosed: true });
const handlers = createAdminGrantsHandlers(deps);
@@ -1281,23 +1320,37 @@ describe('createAdminGrantsHandlers', () => {
await handlers.assignGrant(req, res);
expect(recordAuditEntry).toHaveBeenCalledWith(
- expect.anything(),
+ expect.objectContaining({ action: 'grant.assigned', outcome: 'success' }),
expect.objectContaining({ failClosed: true }),
);
- // compensating rollback removes the grant we just created
- expect(deps.revokeCapability).toHaveBeenCalledWith(
- expect.objectContaining({
- principalType: PrincipalType.ROLE,
- principalId: 'editor',
- capability: SystemCapabilities.READ_USERS,
- }),
- );
+ expect(deps.grantCapability).not.toHaveBeenCalled();
+ expect(deps.revokeCapability).not.toHaveBeenCalled();
+ expect(status).toHaveBeenCalledWith(500);
+ });
+
+ it('a failed grant write leaves a failure entry and surfaces the error', async () => {
+ const recordAuditEntry = jest.fn().mockResolvedValue(undefined);
+ const deps = createDeps({
+ recordAuditEntry,
+ auditFailClosed: true,
+ grantCapability: jest.fn().mockRejectedValue(new Error('write failed')),
+ });
+ const handlers = createAdminGrantsHandlers(deps);
+ const { req, res, status } = createReqRes({ body: assignBody, user: reqUser() });
+
+ await handlers.assignGrant(req, res);
+
+ expect(recordAuditEntry.mock.calls.map(([input]) => input.outcome)).toEqual([
+ 'success',
+ 'failure',
+ ]);
+ expect(deps.revokeCapability).not.toHaveBeenCalled();
expect(status).toHaveBeenCalledWith(500);
});
- it('fail-closed: a failed revoke audit restores the grant and 500s', async () => {
+ it('fail-closed: a failed revoke audit keeps the revoke, retries the entry and returns 200', async () => {
const recordAuditEntry = jest.fn().mockRejectedValue(new Error('audit down'));
- const grantCapability = jest.fn().mockResolvedValue({ grant: { id: 'g1' }, created: true });
+ const grantCapability = jest.fn();
const deps = createDeps({
recordAuditEntry,
auditFailClosed: true,
@@ -1309,15 +1362,117 @@ describe('createAdminGrantsHandlers', () => {
await handlers.revokeGrant(req, res);
- // compensating re-grant restores access removed before the audit failed
- expect(grantCapability).toHaveBeenCalledWith(
- expect.objectContaining({
- principalType: PrincipalType.ROLE,
- principalId: 'editor',
- capability: SystemCapabilities.READ_USERS,
- }),
- );
- expect(status).toHaveBeenCalledWith(500);
+ expect(grantCapability).not.toHaveBeenCalled();
+ expect(recordAuditEntry).toHaveBeenCalledTimes(2);
+ expect(status).toHaveBeenCalledWith(200);
+ });
+
+ describe('grant lease', () => {
+ function memoryLease() {
+ const held = new Map<string, string>();
+ return {
+ acquireResourceLock: jest.fn(async ({ key, ownerId }) => {
+ if (held.has(key)) return false;
+ held.set(key, ownerId);
+ return true;
+ }),
+ renewResourceLock: jest.fn(async ({ key, ownerId }) => held.get(key) === ownerId),
+ releaseResourceLock: jest.fn(async ({ key, ownerId }) => {
+ if (held.get(key) !== ownerId) return false;
+ held.delete(key);
+ return true;
+ }),
+ held,
+ };
+ }
+
+ it('ends with the state named by the last audit entry when a reassert races a revoke', async () => {
+ const lease = memoryLease();
+ const log: string[] = [];
+ let active = true;
+ const delay = () => new Promise((resolve) => setTimeout(resolve, 20));
+ const deps = createDeps({
+ lease,
+ auditFailClosed: true,
+ recordAuditEntry: jest.fn(async (input) => {
+ await delay();
+ log.push(input.action);
+ }),
+ getCapabilitiesForPrincipal: jest.fn(async () => (active ? [mockGrant()] : [])),
+ grantCapability: jest.fn(async () => {
+ active = true;
+ return { grant: mockGrant(), created: false };
+ }),
+ revokeCapability: jest.fn(async () => {
+ const deletedCount = active ? 1 : 0;
+ active = false;
+ return { deletedCount };
+ }),
+ });
+ const handlers = createAdminGrantsHandlers(deps);
+ const assign = createReqRes({ body: assignBody, user: reqUser() });
+ const revoke = createReqRes({
+ params: { ...assignBody },
+ user: reqUser(),
+ });
+
+ await Promise.all([
+ handlers.assignGrant(assign.req, assign.res),
+ handlers.revokeGrant(revoke.req, revoke.res),
+ ]);
+
+ const last = log[log.length - 1];
+ expect(active).toBe(last === 'grant.assigned');
+ expect(lease.held.size).toBe(0);
+ });
+
+ it('uses the lease limits from the loaded librechat.yaml config', async () => {
+ const lease = memoryLease();
+ const deps = createDeps({ lease });
+ const handlers = createAdminGrantsHandlers(deps);
+ const { req, res } = createReqRes({ body: assignBody, user: reqUser() });
+ (req as unknown as { config: unknown }).config = {
+ config: { permissions: { leaseTtlMs: 45_000, leaseWaitMs: 1_500 } },
+ };
+
+ await handlers.assignGrant(req, res);
+
+ expect(lease.acquireResourceLock).toHaveBeenCalledWith(
+ expect.objectContaining({ ttlMs: 45_000 }),
+ );
+ });
+
+ it('answers 409 with the busy code and writes nothing when the lease stays taken', async () => {
+ const lease = memoryLease();
+ lease.acquireResourceLock.mockResolvedValue(false);
+ const deps = createDeps({ lease, auditFailClosed: true });
+ const handlers = createAdminGrantsHandlers(deps);
+ const { req, res, status, json } = createReqRes({ body: assignBody, user: reqUser() });
+
+ await handlers.assignGrant(req, res);
+
+ expect(status).toHaveBeenCalledWith(409);
+ expect(json).toHaveBeenCalledWith(
+ expect.objectContaining({ code: 'RESOURCE_PERMISSIONS_BUSY' }),
+ );
+ expect(deps.grantCapability).not.toHaveBeenCalled();
+ });
+
+ it('aborts the revoke without deleting when the lease was lost', async () => {
+ const lease = memoryLease();
+ lease.renewResourceLock.mockResolvedValue(false);
+ const deps = createDeps({ lease });
+ const handlers = createAdminGrantsHandlers(deps);
+ const { req, res, status } = createReqRes({
+ params: { ...assignBody },
+ user: reqUser(),
+ });
+
+ await handlers.revokeGrant(req, res);
+
+ expect(status).toHaveBeenCalledWith(409);
+ expect(deps.revokeCapability).not.toHaveBeenCalled();
+ });
});
});
});
diff --git a/packages/api/src/admin/grants.ts b/packages/api/src/admin/grants.ts
index bba4495cfb..a8a8c9efd5 100644
--- a/packages/api/src/admin/grants.ts
+++ b/packages/api/src/admin/grants.ts
@@ -17,6 +17,14 @@ import type { Response } from 'express';
import type { Types } from 'mongoose';
import type { ResolvedPrincipal } from '~/types/principal';
import type { ServerRequest } from '~/types/http';
+import type { LeaseDeps } from './lease';
+import {
+ ResourcePermissionsBusyError,
+ ResourcePermissionsUnavailableError,
+} from '~/acl/accessControlService';
+import { PERMISSION_AUDIT_UNAVAILABLE } from '~/acl/codes';
+import { getSafeErrorMetadata } from '~/utils/errors';
+import { grantLeaseKey, withLease } from './lease';
import { parsePagination } from './pagination';
import { buildAuditContext } from './context';
@@ -87,12 +95,13 @@ export interface AdminGrantsDeps {
options?: RecordAuditEntryOptions,
) => Promise<void>;
/**
- * When true, a failed audit write surfaces as a 5xx instead of being
- * swallowed. The grant itself is already persisted (and grant writes are
- * idempotent upserts), so a client retry is safe; an operator must reconcile
- * the missing audit row. Defaults to fail-open.
+ * When true, an assignment is audited before it is written, and a failed audit
+ * write surfaces as a 5xx with nothing granted. A revoke only removes access,
+ * so it always stands and its entry is retried. Defaults to fail-open.
*/
auditFailClosed?: boolean;
+ /** Serializes the audit and write of one grant against concurrent assign/revoke of it. */
+ lease?: LeaseDeps;
}
/** Currently ROLE-only; Record/Set structure preserved for future principal-type expansion. */
@@ -120,8 +129,15 @@ export function createAdminGrantsHandlers(deps: AdminGrantsDeps): {
checkRoleExists,
recordAuditEntry,
auditFailClosed,
+ lease,
} = deps;
+ /** Lease limits come from `permissions.leaseTtlMs` / `leaseWaitMs` in librechat.yaml. */
+ const leaseOptions = (req: ServerRequest) => ({
+ ttlMs: req.config?.config?.permissions?.leaseTtlMs,
+ waitMs: req.config?.config?.permissions?.leaseWaitMs,
+ });
+
async function emitAudit(args: {
action: AuditAction;
caller: { userId: string; actorName: string; role: string; tenantId?: string };
@@ -129,6 +145,10 @@ export function createAdminGrantsHandlers(deps: AdminGrantsDeps): {
principalId: string;
capability: SystemCapability;
context?: AuditContext;
+ /** Never fail the caller on an audit failure, regardless of `auditFailClosed`. */
+ failOpen?: boolean;
+ outcome?: 'success' | 'failure';
+ metadata?: Record<string, unknown>;
}): Promise<void> {
if (!recordAuditEntry) return;
/** The grants surface is ROLE-only today (see `MANAGE_CAPABILITY_BY_TYPE`),
@@ -137,15 +157,15 @@ export function createAdminGrantsHandlers(deps: AdminGrantsDeps): {
* When USER and GROUP grants are enabled, resolve the display name here. */
const input: RecordAuditEntryInput = {
action: args.action,
- outcome: 'success',
+ outcome: args.outcome ?? 'success',
severity: 'warning',
actor: { type: 'user', id: args.caller.userId, name: args.caller.actorName },
target: { type: args.principalType, id: args.principalId, name: args.principalId },
- metadata: { capability: args.capability },
+ metadata: { capability: args.capability, ...args.metadata },
context: args.context,
tenantId: args.caller.tenantId,
};
- if (auditFailClosed) {
+ if (auditFailClosed && !args.failOpen) {
/** Let the failure propagate to the handler (→ 5xx); see `auditFailClosed`. */
await recordAuditEntry(input, { failClosed: true });
return;
@@ -154,7 +174,7 @@ export function createAdminGrantsHandlers(deps: AdminGrantsDeps): {
await recordAuditEntry(input);
} catch (err) {
/** Fail-open: audit failure must not roll back the grant. */
- logger.error('[adminGrants] audit write failed', err);
+ logger.error('[adminGrants] audit write failed', { ...getSafeErrorMetadata(err) });
}
}
@@ -404,45 +424,73 @@ export function createAdminGrantsHandlers(deps: AdminGrantsDeps): {
}
}
- /** `created` comes from the atomic upsert result, so a re-assert of a
- * capability the role already holds (or a concurrent double-assign) is not
- * audited as a new change — and a new tenant-scoped grant is correctly
- * distinguished from an inherited platform-level one. */
- const { grant, created } = await grantCapability({
- principalType,
- principalId,
- capability,
- tenantId,
- grantedBy: caller.userId,
- });
- if (!grant) {
- return res.status(500).json({ error: 'Grant operation returned no result' });
- }
- if (created) {
- try {
- await emitAudit({
- action: 'grant.assigned',
- caller,
+ return await withLease(
+ lease,
+ grantLeaseKey(principalType, principalId, capability, tenantId),
+ async (fence) => {
+ /** A re-assert is still audited: a concurrent revoke can remove the grant between this
+ * read and the write, so skipping it could leave a live grant without an entry. */
+ const existingGrants = await getCapabilitiesForPrincipal({
principalType,
principalId,
- capability,
- context: buildAuditContext(req),
+ tenantId,
});
- } catch (auditErr) {
- /** Fail-closed only (emitAudit rethrows here): roll back the grant we
- * just created so a 5xx never leaves an unaudited grant in place. */
- await revokeCapability({ principalType, principalId, capability, tenantId }).catch((e) =>
- logger.error('[adminGrants] compensating revoke after audit failure failed', e),
+ const reassert = existingGrants.some(
+ (existing) => existing.capability === capability && existing.tenantId == tenantId,
);
- logger.error(
- '[adminGrants] assignGrant audit failed (fail-closed) — rolled back grant',
- auditErr,
- );
- return res.status(500).json({ error: 'Failed to record audit entry' });
- }
- }
- return res.status(201).json({ grant });
+ const auditAssignment = (outcome: 'success' | 'failure') =>
+ emitAudit({
+ action: 'grant.assigned',
+ caller,
+ principalType,
+ principalId,
+ capability,
+ context: buildAuditContext(req),
+ outcome,
+ failOpen: outcome === 'failure',
+ metadata: reassert ? { reassert: true } : undefined,
+ });
+ /** Audit first: with fail-closed auditing a grant that cannot be recorded is never written. */
+ try {
+ await auditAssignment('success');
+ } catch (auditErr) {
+ logger.error('[adminGrants] assignGrant audit failed (fail-closed), nothing granted', {
+ ...getSafeErrorMetadata(auditErr),
+ });
+ return res
+ .status(500)
+ .json({ error: 'Failed to record audit entry', code: PERMISSION_AUDIT_UNAVAILABLE });
+ }
+ let result: Awaited<ReturnType<typeof grantCapability>>;
+ try {
+ await fence();
+ result = await grantCapability({
+ principalType,
+ principalId,
+ capability,
+ tenantId,
+ grantedBy: caller.userId,
+ });
+ } catch (writeErr) {
+ await auditAssignment('failure');
+ throw writeErr;
+ }
+ const { grant } = result;
+ if (!grant) {
+ await auditAssignment('failure');
+ return res.status(500).json({ error: 'Grant operation returned no result' });
+ }
+ return res.status(201).json({ grant });
+ },
+ leaseOptions(req),
+ );
} catch (error) {
+ if (
+ error instanceof ResourcePermissionsBusyError ||
+ error instanceof ResourcePermissionsUnavailableError
+ ) {
+ return res.status(error.statusCode).json({ error: error.message, code: error.code });
+ }
logger.error('[adminGrants] assignGrant error:', error);
return res.status(500).json({ error: 'Failed to assign grant' });
}
@@ -481,48 +529,59 @@ export function createAdminGrantsHandlers(deps: AdminGrantsDeps): {
return res.status(403).json({ error: 'Insufficient permissions' });
}
- const revokeResult = await revokeCapability({
- principalType: principalType as PrincipalType,
- principalId,
- capability: capability as SystemCapability,
- tenantId,
- });
- // Only emit the audit entry when a grant document was actually deleted —
- // a no-op revoke (the grant never existed) must not appear in the audit
- // trail or the forensic record becomes misleading.
- if (revokeResult.deletedCount > 0) {
- try {
- await emitAudit({
- action: 'grant.removed',
- caller,
- principalType: principalType as PrincipalType,
- principalId,
- capability: capability as SystemCapability,
- context: buildAuditContext(req),
- });
- } catch (auditErr) {
- /** Fail-closed only (emitAudit rethrows here): restore the grant we
- * just removed so a 5xx never leaves an unaudited revoke. The original
- * `grantedBy`/`grantedAt` are not recoverable here; restoring access is
- * the priority on this rare error path. */
- await grantCapability({
+ return await withLease(
+ lease,
+ grantLeaseKey(principalType, principalId, capability, tenantId),
+ async (fence) => {
+ await fence();
+ const revokeResult = await revokeCapability({
principalType: principalType as PrincipalType,
principalId,
capability: capability as SystemCapability,
tenantId,
- grantedBy: caller.userId,
- }).catch((e) =>
- logger.error('[adminGrants] compensating re-grant after audit failure failed', e),
- );
- logger.error(
- '[adminGrants] revokeGrant audit failed (fail-closed) — restored grant',
- auditErr,
- );
- return res.status(500).json({ error: 'Failed to record audit entry' });
- }
- }
- return res.status(200).json({ success: true });
+ });
+ // Only emit the audit entry when a grant document was actually deleted —
+ // a no-op revoke (the grant never existed) must not appear in the audit
+ // trail or the forensic record becomes misleading.
+ if (revokeResult.deletedCount > 0) {
+ try {
+ await emitAudit({
+ action: 'grant.removed',
+ caller,
+ principalType: principalType as PrincipalType,
+ principalId,
+ capability: capability as SystemCapability,
+ context: buildAuditContext(req),
+ });
+ } catch (auditErr) {
+ /** A revoke only removes access, so an audit outage must not keep it live:
+ * the revoke stands, the failure is logged with safe metadata, and the
+ * entry is attempted once more without failing the request. */
+ logger.error('[adminGrants] revokeGrant audit failed, revoke kept', {
+ ...getSafeErrorMetadata(auditErr),
+ });
+ await emitAudit({
+ action: 'grant.removed',
+ caller,
+ principalType: principalType as PrincipalType,
+ principalId,
+ capability: capability as SystemCapability,
+ context: buildAuditContext(req),
+ failOpen: true,
+ });
+ }
+ }
+ return res.status(200).json({ success: true });
+ },
+ leaseOptions(req),
+ );
} catch (error) {
+ if (
+ error instanceof ResourcePermissionsBusyError ||
+ error instanceof ResourcePermissionsUnavailableError
+ ) {
+ return res.status(error.statusCode).json({ error: error.message, code: error.code });
+ }
logger.error('[adminGrants] revokeGrant error:', error);
return res.status(500).json({ error: 'Failed to revoke grant' });
}
diff --git a/packages/api/src/admin/index.ts b/packages/api/src/admin/index.ts
index 0ecb60212c..2e72421907 100644
--- a/packages/api/src/admin/index.ts
+++ b/packages/api/src/admin/index.ts
@@ -9,7 +9,12 @@ export { createAdminAuditLogHandlers } from './auditLog';
export { createAdminCodeEnvironmentHandlers } from './code';
export { buildAuditContext } from './context';
export { cascadeStep, runCascade } from './cascade';
-export { createAdminAuditEmitter, recordSystemAuditEntry, diffRolePermissions } from './audit';
+export {
+ createAdminAuditEmitter,
+ recordSystemAuditEntry,
+ diffRolePermissions,
+ isAuditFailClosed,
+} from './audit';
export { resolveConfigSecret, redactConfigSecretMaps } from './secrets';
export type { AdminConfigDeps } from './config';
export type { AdminLangfuseDeps } from './langfuse';
diff --git a/packages/api/src/admin/lease.spec.ts b/packages/api/src/admin/lease.spec.ts
new file mode 100644
index 0000000000..5a2240f869
--- /dev/null
+++ b/packages/api/src/admin/lease.spec.ts
@@ -0,0 +1,104 @@
+import { inspect } from 'util';
+import { logger } from '@librechat/data-schemas';
+import { grantLeaseKey, withLease } from './lease';
+
+jest.mock('@librechat/data-schemas', () => ({
+ ...jest.requireActual('@librechat/data-schemas'),
+ logger: { error: jest.fn(), warn: jest.fn(), info: jest.fn(), debug: jest.fn() },
+}));
+
+describe('withLease', () => {
+ afterEach(() => jest.useRealTimers());
+
+ it('renews the lease while the callback is still running', async () => {
+ jest.useFakeTimers();
+ const lease = {
+ acquireResourceLock: jest.fn().mockResolvedValue(true),
+ renewResourceLock: jest.fn().mockResolvedValue(true),
+ releaseResourceLock: jest.fn().mockResolvedValue(true),
+ };
+ const run = withLease(
+ lease,
+ 'k',
+ () => new Promise<string>((resolve) => setTimeout(() => resolve('done'), 25_000)),
+ );
+
+ await jest.advanceTimersByTimeAsync(25_000);
+
+ await expect(run).resolves.toBe('done');
+ expect(lease.renewResourceLock.mock.calls.length).toBeGreaterThanOrEqual(2);
+ expect(lease.releaseResourceLock).toHaveBeenCalledTimes(1);
+ });
+
+ it('does not lose the lease on a transient renewal error', async () => {
+ const lease = {
+ acquireResourceLock: jest.fn().mockResolvedValue(true),
+ renewResourceLock: jest.fn().mockRejectedValueOnce(new Error('blip')).mockResolvedValue(true),
+ releaseResourceLock: jest.fn().mockResolvedValue(true),
+ };
+ await expect(
+ withLease(lease, 'k', async (fence) => {
+ await expect(fence()).rejects.toMatchObject({ code: 'RESOURCE_PERMISSIONS_UNAVAILABLE' });
+ await fence();
+ return 'ok';
+ }),
+ ).resolves.toBe('ok');
+ });
+
+ it('turns a lock-store failure on acquire into the unavailable error', async () => {
+ const lease = {
+ acquireResourceLock: jest.fn().mockRejectedValue(new Error('mongo secret detail')),
+ renewResourceLock: jest.fn(),
+ releaseResourceLock: jest.fn(),
+ };
+ await expect(withLease(lease, 'k', async () => 'x')).rejects.toMatchObject({
+ statusCode: 503,
+ message: expect.not.stringContaining('secret'),
+ });
+ });
+
+ it('honors configured wait and TTL limits', async () => {
+ const lease = {
+ acquireResourceLock: jest.fn().mockResolvedValue(false),
+ renewResourceLock: jest.fn().mockResolvedValue(true),
+ releaseResourceLock: jest.fn().mockResolvedValue(true),
+ };
+ const started = Date.now();
+ await expect(withLease(lease, 'k', async () => 'x', { waitMs: 0 })).rejects.toMatchObject({
+ code: 'RESOURCE_PERMISSIONS_BUSY',
+ });
+ expect(Date.now() - started).toBeLessThan(1000);
+
+ lease.acquireResourceLock.mockResolvedValue(true);
+ await withLease(lease, 'k', async () => 'x', { ttlMs: 12_345 });
+ expect(lease.acquireResourceLock).toHaveBeenLastCalledWith(
+ expect.objectContaining({ ttlMs: 12_345 }),
+ );
+ });
+
+ it('logs a failed release with safe metadata and still returns the result', async () => {
+ const lease = {
+ acquireResourceLock: jest.fn().mockResolvedValue(true),
+ renewResourceLock: jest.fn().mockResolvedValue(true),
+ releaseResourceLock: jest.fn().mockRejectedValue(new Error('mongo secret detail')),
+ };
+ (logger.error as jest.Mock).mockClear();
+
+ await expect(withLease(lease, 'k', async () => 'ok')).resolves.toBe('ok');
+
+ expect(logger.error).toHaveBeenCalledTimes(1);
+ expect(inspect((logger.error as jest.Mock).mock.calls)).not.toContain('secret');
+ });
+
+ it('keys one grant the same for a user ObjectId in either case', () => {
+ const hex = 'abcdef0123456789abcdef01';
+ expect(grantLeaseKey('user', hex, 'cap')).toBe(grantLeaseKey('USER', hex.toUpperCase(), 'cap'));
+ });
+
+ it('keeps role names case-sensitive', () => {
+ const hex = 'abcdef0123456789abcdef01';
+ expect(grantLeaseKey('role', hex, 'cap')).not.toBe(
+ grantLeaseKey('role', hex.toUpperCase(), 'cap'),
+ );
+ });
+});
diff --git a/packages/api/src/admin/lease.ts b/packages/api/src/admin/lease.ts
new file mode 100644
index 0000000000..9b31461a1e
--- /dev/null
+++ b/packages/api/src/admin/lease.ts
@@ -0,0 +1,94 @@
+import { Types } from 'mongoose';
+import { logger } from '@librechat/data-schemas';
+import { PrincipalType } from 'librechat-data-provider';
+import {
+ ResourcePermissionsBusyError,
+ ResourcePermissionsUnavailableError,
+} from '~/acl/accessControlService';
+import { getSafeErrorMetadata } from '~/utils/errors';
+
+export interface LeaseDeps {
+ acquireResourceLock: (data: {
+ key: string;
+ ownerId: string;
+ ttlMs: number;
+ readyWaitMs?: number;
+ }) => Promise<boolean>;
+ renewResourceLock: (data: { key: string; ownerId: string; ttlMs: number }) => Promise<boolean>;
+ releaseResourceLock: (data: { key: string; ownerId: string }) => Promise<boolean>;
+}
+
+export type LeaseOptions = { ttlMs?: number; waitMs?: number };
+
+const LEASE_TTL_MS = 30_000;
+const LEASE_WAIT_MS = 3_000;
+const LEASE_POLL_MS = 50;
+
+/** One key per grant, with ObjectId hex case-folded the way the ACL lease key is. */
+export function grantLeaseKey(
+ principalType: string,
+ principalId: string,
+ capability: string,
+ tenantId?: string,
+): string {
+ /** Role names are case-sensitive strings; only ObjectId-backed principals are canonicalized. */
+ const id =
+ principalType !== PrincipalType.ROLE && Types.ObjectId.isValid(principalId)
+ ? new Types.ObjectId(principalId).toString()
+ : principalId;
+ return `grant:${principalType.toLowerCase()}:${id}:${capability}:${tenantId ?? ''}`;
+}
+
+/** A lock-store failure is a retryable 503, never a raw driver error. */
+const unavailable = (): never => {
+ throw new ResourcePermissionsUnavailableError();
+};
+
+/**
+ * Runs `fn` while holding a lease on `key`, so the audit entry and the write of one
+ * grant are ordered against a concurrent change of the same grant. `fence` re-validates
+ * and extends the lease and throws `ResourcePermissionsBusyError` once it was lost.
+ * Without lease deps `fn` runs unserialized.
+ */
+export async function withLease<T>(
+ deps: LeaseDeps | undefined,
+ key: string,
+ fn: (fence: () => Promise<void>) => Promise<T>,
+ { ttlMs = LEASE_TTL_MS, waitMs = LEASE_WAIT_MS }: LeaseOptions = {},
+): Promise<T> {
+ if (!deps) {
+ return fn(async () => undefined);
+ }
+ const lock = { key, ownerId: new Types.ObjectId().toString() };
+ const deadline = Date.now() + waitMs;
+ while (
+ !(await deps
+ .acquireResourceLock({ ...lock, ttlMs, readyWaitMs: Math.max(0, deadline - Date.now()) })
+ .catch(unavailable))
+ ) {
+ if (Date.now() >= deadline) {
+ throw new ResourcePermissionsBusyError();
+ }
+ await new Promise((resolve) => setTimeout(resolve, LEASE_POLL_MS));
+ }
+ let lost = false;
+ const renew = async () => {
+ /** Only a definitive "not the holder" answer loses the lease; a transient error may be retried. */
+ if (lost || !(await deps.renewResourceLock({ ...lock, ttlMs }).catch(unavailable))) {
+ lost = true;
+ throw new ResourcePermissionsBusyError();
+ }
+ };
+ /** Keeps the lease alive through slow audit writes between fences. */
+ const heartbeat = setInterval(() => void renew().catch(() => undefined), ttlMs / 3);
+ try {
+ return await fn(renew);
+ } finally {
+ clearInterval(heartbeat);
+ await deps.releaseResourceLock(lock).catch((error) =>
+ logger.error('[adminGrants] Lease release failed; it expires on its own', {
+ ...getSafeErrorMetadata(error),
+ }),
+ );
+ }
+}
diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts
index 54ba5cd185..f501999978 100644
--- a/packages/api/src/index.ts
+++ b/packages/api/src/index.ts
@@ -1,6 +1,7 @@
export * from './app';
export * from './acl/accessControlService';
export * from './acl/insightsPermissions';
+export * from './acl/audit';
export * from './acl/middleware';
export * from './acl/principals';
export * from './acl/search';
diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts
index f216aa5549..e54e08cd23 100644
--- a/packages/data-provider/src/config.ts
+++ b/packages/data-provider/src/config.ts
@@ -4021,6 +4021,12 @@ export type TChatProjectsConfig = z.infer<typeof chatProjectsConfigSchema>;
/** Maximum CAS attempts per ACL document, including the initial attempt. */
export const permissionWriteAttemptsSchema = z.number().int().min(1).max(100).default(3);
+/** How long a holder keeps the per-resource permission lease without renewing it. */
+export const permissionLeaseTtlSchema = z.number().int().min(1000).max(600000).default(30000);
+
+/** How long a permission update waits for a contended per-resource lease before answering 409. */
+export const permissionLeaseWaitSchema = z.number().int().min(0).max(60000).default(3000);
+
/**
* Validation limits for the conversation list's filter facets. The field defaults are the
* only definition of these values: `getConfigDefaults()` resolves them for AppService, and
@@ -4057,7 +4063,13 @@ export type TToolCallPreviewsConfig = z.infer<typeof toolCallPreviewsConfigSchem
export const configSchema = z.object({
version: z.string(),
- permissions: z.object({ maxWriteAttempts: permissionWriteAttemptsSchema }).optional(),
+ permissions: z
+ .object({
+ maxWriteAttempts: permissionWriteAttemptsSchema,
+ leaseTtlMs: permissionLeaseTtlSchema,
+ leaseWaitMs: permissionLeaseWaitSchema,
+ })
+ .optional(),
cache: z.boolean().default(true),
projects: chatProjectsConfigSchema,
ocr: ocrSchema.optional(),
diff --git a/packages/data-provider/src/permissionWriteAttempts.spec.ts b/packages/data-provider/src/permissionWriteAttempts.spec.ts
index 1458ec85b6..90dc195c1d 100644
--- a/packages/data-provider/src/permissionWriteAttempts.spec.ts
+++ b/packages/data-provider/src/permissionWriteAttempts.spec.ts
@@ -5,12 +5,32 @@ describe('permission write retry budget', () => {
expect(permissionWriteAttemptsSchema.parse(undefined)).toBe(3);
expect(configSchema.parse({ version: '1.3.1', permissions: {} }).permissions).toEqual({
maxWriteAttempts: 3,
+ leaseTtlMs: 30000,
+ leaseWaitMs: 3000,
});
expect(
configSchema.parse({ version: '1.3.1', permissions: { maxWriteAttempts: 7 } }).permissions,
- ).toEqual({ maxWriteAttempts: 7 });
+ ).toEqual({ maxWriteAttempts: 7, leaseTtlMs: 30000, leaseWaitMs: 3000 });
+ expect(
+ configSchema.parse({
+ version: '1.3.1',
+ permissions: { leaseTtlMs: 60000, leaseWaitMs: 5000 },
+ }).permissions,
+ ).toEqual({ maxWriteAttempts: 3, leaseTtlMs: 60000, leaseWaitMs: 5000 });
});
test.each([NaN, Infinity, -1, 0, 1.5, 101])('rejects %s', (value) => {
expect(permissionWriteAttemptsSchema.safeParse(value).success).toBe(false);
});
});
+
+describe('permission lease limits', () => {
+ test.each([
+ [{ leaseTtlMs: 999 }],
+ [{ leaseTtlMs: 600001 }],
+ [{ leaseWaitMs: -1 }],
+ [{ leaseWaitMs: 60001 }],
+ [{ leaseTtlMs: 1.5 }],
+ ])('rejects %j', (permissions) => {
+ expect(configSchema.safeParse({ version: '1.3.1', permissions }).success).toBe(false);
+ });
+});
diff --git a/packages/data-schemas/src/methods/aclEntry.spec.ts b/packages/data-schemas/src/methods/aclEntry.spec.ts
index fc123a68bf..a2dcbcf1c3 100644
--- a/packages/data-schemas/src/methods/aclEntry.spec.ts
+++ b/packages/data-schemas/src/methods/aclEntry.spec.ts
@@ -207,6 +207,25 @@ describe('AclEntry Model Tests', () => {
expect(entries).toHaveLength(2);
});
+ test('reads mutation snapshots from the primary even when a session is supplied', async () => {
+ const read = jest.spyOn(mongoose.Query.prototype, 'read');
+ const session = await mongoose.startSession();
+ try {
+ const entries = await methods.findEntriesByPrincipalsAndResource(
+ [{ principalType: PrincipalType.USER, principalId: userId }],
+ ResourceType.AGENT,
+ resourceId,
+ session,
+ true,
+ );
+ expect(entries).toHaveLength(1);
+ expect(read).toHaveBeenCalledWith('primary');
+ } finally {
+ read.mockRestore();
+ await session.endSession();
+ }
+ });
+
test('should check if user has permission', async () => {
const principalsList = [{ principalType: PrincipalType.USER, principalId: userId }];
diff --git a/packages/data-schemas/src/methods/aclEntry.ts b/packages/data-schemas/src/methods/aclEntry.ts
index 0173725175..fdbbb71c79 100644
--- a/packages/data-schemas/src/methods/aclEntry.ts
+++ b/packages/data-schemas/src/methods/aclEntry.ts
@@ -142,6 +142,8 @@ export function createAclEntryMethods(mongoose: typeof import('mongoose')): {
principalsList: Array<{ principalType: string; principalId?: string | Types.ObjectId }>,
resourceType: string,
resourceId: string | Types.ObjectId,
+ session?: ClientSession,
+ readPrimary?: boolean,
) => Promise<IAclEntry[]>;
hasPermission: (
principalsList: Array<{ principalType: string; principalId?: string | Types.ObjectId }>,
@@ -272,6 +274,8 @@ export function createAclEntryMethods(mongoose: typeof import('mongoose')): {
principalsList: Array<{ principalType: string; principalId?: string | Types.ObjectId }>,
resourceType: string,
resourceId: string | Types.ObjectId,
+ session?: ClientSession,
+ readPrimary?: boolean,
): Promise<IAclEntry[]> {
const AclEntry = mongoose.models.AclEntry as Model<IAclEntry>;
const principalsQuery = principalsList.map((p) => ({
@@ -279,11 +283,19 @@ export function createAclEntryMethods(mongoose: typeof import('mongoose')): {
...(p.principalType !== PrincipalType.PUBLIC && { principalId: p.principalId }),
}));
- return await AclEntry.find({
+ const query = AclEntry.find({
$or: principalsQuery,
resourceType,
resourceId,
- }).lean<IAclEntry[]>();
+ });
+ if (session) {
+ query.session(session);
+ }
+ if (readPrimary && !session?.inTransaction()) {
+ /** Mutation snapshots must not come from a lagging secondary; a transaction already reads from the primary. */
+ query.read('primary');
+ }
+ return await query.lean<IAclEntry[]>();
}
/**
diff --git a/packages/data-schemas/src/methods/index.ts b/packages/data-schemas/src/methods/index.ts
index 49c4df8e59..8be120e86d 100644
--- a/packages/data-schemas/src/methods/index.ts
+++ b/packages/data-schemas/src/methods/index.ts
@@ -18,6 +18,7 @@ import {
createOpenIDRefreshFlightMethods,
type OpenIDRefreshFlightMethods,
} from './openidRefreshFlight';
+import { createResourceLockMethods, type ResourceLockMethods } from './resourceLock';
export {
createMCPAuthorizationFenceRetryStorage,
type MCPAuthorizationFenceRetryStorage,
@@ -256,6 +257,7 @@ export type AllMethods = ToolApprovalGrantStorage &
TokenListMethods &
RefreshTokenBridgeMethods &
OpenIDRefreshFlightMethods &
+ ResourceLockMethods &
PasskeyMethods &
RoleMethods &
KeyMethods &
@@ -495,6 +497,7 @@ export function createMethods(
...createTokenListMethods(mongoose),
...createRefreshTokenBridgeMethods(mongoose),
...createOpenIDRefreshFlightMethods(mongoose),
+ ...createResourceLockMethods(mongoose),
...createPasskeyMethods(mongoose),
...roleMethods,
...createKeyMethods(mongoose),
@@ -563,6 +566,7 @@ export type {
TokenMethods,
RefreshTokenBridgeMethods,
OpenIDRefreshFlightMethods,
+ ResourceLockMethods,
RoleMethods,
KeyMethods,
MemoryMethods,
diff --git a/packages/data-schemas/src/methods/resourceLock.spec.ts b/packages/data-schemas/src/methods/resourceLock.spec.ts
new file mode 100644
index 0000000000..7908971f2e
--- /dev/null
+++ b/packages/data-schemas/src/methods/resourceLock.spec.ts
@@ -0,0 +1,80 @@
+import mongoose from 'mongoose';
+import { MongoMemoryServer } from 'mongodb-memory-server';
+import type * as t from '~/types';
+import { createResourceLockMethods } from './resourceLock';
+import resourceLockSchema from '~/schema/resourceLock';
+
+let mongoServer: MongoMemoryServer;
+let methods: ReturnType<typeof createResourceLockMethods>;
+
+beforeAll(async () => {
+ mongoServer = await MongoMemoryServer.create();
+ await mongoose.connect(mongoServer.getUri());
+ if (!mongoose.models.ResourceLock) {
+ mongoose.model<t.IResourceLock>('ResourceLock', resourceLockSchema);
+ }
+});
+
+afterAll(async () => {
+ await mongoose.disconnect();
+ await mongoServer?.stop();
+});
+
+beforeEach(async () => {
+ await mongoose.connection.dropDatabase();
+ methods = createResourceLockMethods(mongoose);
+});
+
+describe('resource locks', () => {
+ it('grants the lease to one owner at a time', async () => {
+ const key = 'acl:agent:1';
+ expect(await methods.acquireResourceLock({ key, ownerId: 'a', ttlMs: 30_000 })).toBe(true);
+ expect(await methods.acquireResourceLock({ key, ownerId: 'b', ttlMs: 30_000 })).toBe(false);
+ expect(
+ await methods.acquireResourceLock({ key: 'acl:agent:2', ownerId: 'b', ttlMs: 30_000 }),
+ ).toBe(true);
+ });
+
+ it('lets another owner take over a lease its crashed holder left to expire', async () => {
+ const key = 'acl:agent:1';
+ await methods.acquireResourceLock({ key, ownerId: 'crashed', ttlMs: 1 });
+ await new Promise((resolve) => setTimeout(resolve, 20));
+
+ expect(await methods.acquireResourceLock({ key, ownerId: 'b', ttlMs: 30_000 })).toBe(true);
+ expect(await methods.releaseResourceLock({ key, ownerId: 'crashed' })).toBe(false);
+ });
+
+ it('ignores a release from a non-owner and frees the lease for the owner', async () => {
+ const key = 'acl:agent:1';
+ await methods.acquireResourceLock({ key, ownerId: 'a', ttlMs: 30_000 });
+
+ expect(await methods.releaseResourceLock({ key, ownerId: 'b' })).toBe(false);
+ expect(await methods.acquireResourceLock({ key, ownerId: 'b', ttlMs: 30_000 })).toBe(false);
+ expect(await methods.releaseResourceLock({ key, ownerId: 'a' })).toBe(true);
+ expect(await methods.acquireResourceLock({ key, ownerId: 'b', ttlMs: 30_000 })).toBe(true);
+ });
+
+ it('renews only for the live holder', async () => {
+ const key = 'acl:agent:1';
+ await methods.acquireResourceLock({ key, ownerId: 'a', ttlMs: 30_000 });
+ expect(await methods.renewResourceLock({ key, ownerId: 'a', ttlMs: 30_000 })).toBe(true);
+ expect(await methods.renewResourceLock({ key, ownerId: 'b', ttlMs: 30_000 })).toBe(false);
+ await methods.acquireResourceLock({ key: 'acl:agent:2', ownerId: 'c', ttlMs: 1 });
+ await new Promise((resolve) => setTimeout(resolve, 5));
+ expect(
+ await methods.renewResourceLock({ key: 'acl:agent:2', ownerId: 'c', ttlMs: 30_000 }),
+ ).toBe(false);
+ });
+
+ it('never moves the expiry backwards on an overlapping renewal', async () => {
+ const key = 'acl:agent:3';
+ await methods.acquireResourceLock({ key, ownerId: 'a', ttlMs: 60_000 });
+ const before = (await mongoose.models.ResourceLock.findOne({ key }).lean<{
+ expiresAt: Date;
+ }>())!.expiresAt;
+ await methods.renewResourceLock({ key, ownerId: 'a', ttlMs: 1_000 });
+ const after = (await mongoose.models.ResourceLock.findOne({ key }).lean<{ expiresAt: Date }>())!
+ .expiresAt;
+ expect(after.getTime()).toBeGreaterThanOrEqual(before.getTime());
+ });
+});
diff --git a/packages/data-schemas/src/methods/resourceLock.ts b/packages/data-schemas/src/methods/resourceLock.ts
new file mode 100644
index 0000000000..b85e25946e
--- /dev/null
+++ b/packages/data-schemas/src/methods/resourceLock.ts
@@ -0,0 +1,114 @@
+import type { Model } from 'mongoose';
+import type { IResourceLock, ResourceLockAcquireData, ResourceLockReleaseData } from '~/types';
+import { createIndexesWithRetry } from '~/utils/retry';
+
+/** A stuck index build must surface as an error instead of hanging the request. */
+const INDEX_BUILD_WAIT_MS = 2_000;
+
+function isDuplicateKeyError(error: unknown): boolean {
+ return typeof error === 'object' && error != null && (error as { code?: unknown }).code === 11000;
+}
+
+export function createResourceLockMethods(mongoose: typeof import('mongoose')): {
+ acquireResourceLock: (data: ResourceLockAcquireData) => Promise<boolean>;
+ renewResourceLock: (data: ResourceLockAcquireData) => Promise<boolean>;
+ releaseResourceLock: (data: ResourceLockReleaseData) => Promise<boolean>;
+} {
+ let indexesPromise: Promise<void> | null = null;
+
+ function ensureIndexes(): Promise<void> {
+ if (!indexesPromise) {
+ const ResourceLock = mongoose.models.ResourceLock as Model<IResourceLock>;
+ indexesPromise = createIndexesWithRetry(ResourceLock, {
+ peerBuildDeadlineMs: INDEX_BUILD_WAIT_MS,
+ }).catch((error) => {
+ indexesPromise = null;
+ throw error;
+ });
+ }
+ return indexesPromise;
+ }
+
+ /** Resolves false instead of waiting past the deadline on a stuck index build. */
+ async function indexesReady(maxWaitMs: number): Promise<boolean> {
+ let timer: NodeJS.Timeout | undefined;
+ const timeout = new Promise<false>((resolve) => {
+ timer = setTimeout(() => resolve(false), maxWaitMs);
+ });
+ try {
+ return await Promise.race([ensureIndexes().then(() => true as const), timeout]);
+ } finally {
+ clearTimeout(timer);
+ }
+ }
+
+ /**
+ * Expiry is computed on the application clock ($$NOW pipelines are avoided for DocumentDB
+ * compatibility). Clock skew between instances only affects liveness: every write is fenced by
+ * `renewResourceLock`, which succeeds only for the current owner token, so a skew-induced early
+ * takeover changes the owner and the previous holder's next renewal fails and aborts with a 409
+ * instead of writing. Exclusivity therefore holds; the delay a skew can cause is bounded by the
+ * skew relative to the TTL. Tracked as a liveness improvement in berry-13/LibreChat#354.
+ *
+ * Takes the lease for `key`, or takes over one whose holder let it expire.
+ * Returns false while another owner holds a live lease.
+ */
+ async function acquireResourceLock({
+ key,
+ ownerId,
+ ttlMs,
+ readyWaitMs,
+ }: ResourceLockAcquireData): Promise<boolean> {
+ const ResourceLock = mongoose.models.ResourceLock as Model<IResourceLock>;
+ if (!(await indexesReady(Math.min(readyWaitMs ?? INDEX_BUILD_WAIT_MS, INDEX_BUILD_WAIT_MS)))) {
+ return false;
+ }
+ const now = new Date();
+ const expiresAt = new Date(now.getTime() + ttlMs);
+ try {
+ await ResourceLock.create({ key, ownerId, expiresAt });
+ return true;
+ } catch (error) {
+ if (!isDuplicateKeyError(error)) {
+ throw error;
+ }
+ }
+ const taken = await ResourceLock.findOneAndUpdate(
+ { key, expiresAt: { $lte: now } },
+ { $set: { ownerId, expiresAt } },
+ { new: true },
+ ).lean();
+ return taken != null;
+ }
+
+ /**
+ * Extends the lease only while `ownerId` still holds it unexpired. A false result
+ * means it was lost (expired or taken over) and the holder must not proceed.
+ */
+ async function renewResourceLock({
+ key,
+ ownerId,
+ ttlMs,
+ }: ResourceLockAcquireData): Promise<boolean> {
+ const ResourceLock = mongoose.models.ResourceLock as Model<IResourceLock>;
+ const now = new Date();
+ const renewed = await ResourceLock.findOneAndUpdate(
+ { key, ownerId, expiresAt: { $gt: now } },
+ /** $max keeps overlapping renewals from moving the expiry backwards. */
+ { $max: { expiresAt: new Date(now.getTime() + ttlMs) } },
+ { new: true },
+ ).lean();
+ return renewed != null;
+ }
+
+ /** Releases the lease only for the owner that holds it; anyone else is a no-op. */
+ async function releaseResourceLock({ key, ownerId }: ResourceLockReleaseData): Promise<boolean> {
+ const ResourceLock = mongoose.models.ResourceLock as Model<IResourceLock>;
+ const result = await ResourceLock.deleteOne({ key, ownerId });
+ return result.deletedCount > 0;
+ }
+
+ return { acquireResourceLock, renewResourceLock, releaseResourceLock };
+}
+
+export type ResourceLockMethods = ReturnType<typeof createResourceLockMethods>;
diff --git a/packages/data-schemas/src/models/index.ts b/packages/data-schemas/src/models/index.ts
index 646f010c55..1d999c870f 100644
--- a/packages/data-schemas/src/models/index.ts
+++ b/packages/data-schemas/src/models/index.ts
@@ -12,6 +12,7 @@ import { createSkillSyncStatusModel } from './skillSyncStatus';
import { createConversationTagModel } from './conversationTag';
import { createCodeEnvironmentModel } from './codeEnvironment';
import { createAgentCategoryModel } from './agentCategory';
+import { createResourceLockModel } from './resourceLock';
import { createChatProjectModel } from './chatProject';
import { createAgentApiKeyModel } from './agentApiKey';
import { createTransactionModel } from './transaction';
@@ -101,6 +102,7 @@ export function createModels(mongoose: typeof import('mongoose')): {
ScheduleRun: ReturnType<typeof createScheduleRunModel>;
RefreshTokenBridge: ReturnType<typeof createRefreshTokenBridgeModel>;
OpenIDRefreshFlight: ReturnType<typeof createOpenIDRefreshFlightModel>;
+ ResourceLock: ReturnType<typeof createResourceLockModel>;
} {
const models = {
User: createUserModel(mongoose),
@@ -152,6 +154,7 @@ export function createModels(mongoose: typeof import('mongoose')): {
ScheduleRun: createScheduleRunModel(mongoose),
RefreshTokenBridge: createRefreshTokenBridgeModel(mongoose),
OpenIDRefreshFlight: createOpenIDRefreshFlightModel(mongoose),
+ ResourceLock: createResourceLockModel(mongoose),
};
/**
* Background index builds fail silently unless an 'index' listener is
diff --git a/packages/data-schemas/src/models/resourceLock.ts b/packages/data-schemas/src/models/resourceLock.ts
new file mode 100644
index 0000000000..91711ffe81
--- /dev/null
+++ b/packages/data-schemas/src/models/resourceLock.ts
@@ -0,0 +1,16 @@
+import { Model } from 'mongoose';
+import type * as t from '~/types';
+import resourceLockSchema from '~/schema/resourceLock';
+
+/**
+ * Short-lived per-resource leases that serialize read-audit-write sequences on
+ * deployments without transactions. Documents expire via TTL.
+ */
+export function createResourceLockModel(
+ mongoose: typeof import('mongoose'),
+): Model<t.IResourceLock> {
+ return (
+ mongoose.models.ResourceLock ||
+ mongoose.model<t.IResourceLock>('ResourceLock', resourceLockSchema)
+ );
+}
diff --git a/packages/data-schemas/src/schema/index.ts b/packages/data-schemas/src/schema/index.ts
index f1d87563f9..d8604b3212 100644
--- a/packages/data-schemas/src/schema/index.ts
+++ b/packages/data-schemas/src/schema/index.ts
@@ -19,6 +19,7 @@ export { default as presetSchema } from './preset';
export { default as promptSchema } from './prompt';
export { default as promptGroupSchema } from './promptGroup';
export { default as openidRefreshFlightSchema } from './openidRefreshFlight';
+export { default as resourceLockSchema } from './resourceLock';
export { default as refreshTokenBridgeSchema } from './refreshTokenBridge';
export { default as roleSchema } from './role';
export { default as passkeySchema } from './passkey';
diff --git a/packages/data-schemas/src/schema/resourceLock.ts b/packages/data-schemas/src/schema/resourceLock.ts
new file mode 100644
index 0000000000..146c02ee08
--- /dev/null
+++ b/packages/data-schemas/src/schema/resourceLock.ts
@@ -0,0 +1,22 @@
+import { Schema } from 'mongoose';
+import type { IResourceLock } from '~/types';
+
+const resourceLockSchema: Schema<IResourceLock> = new Schema<IResourceLock>({
+ key: {
+ type: String,
+ required: true,
+ unique: true,
+ },
+ ownerId: {
+ type: String,
+ required: true,
+ },
+ expiresAt: {
+ type: Date,
+ required: true,
+ },
+});
+
+resourceLockSchema.index({ expiresAt: 1 }, { expireAfterSeconds: 0 });
+
+export default resourceLockSchema;
diff --git a/packages/data-schemas/src/types/index.ts b/packages/data-schemas/src/types/index.ts
index e8a8e9b2fa..eba6570762 100644
--- a/packages/data-schemas/src/types/index.ts
+++ b/packages/data-schemas/src/types/index.ts
@@ -8,6 +8,7 @@ export * from './user';
export * from './token';
export * from './refreshTokenBridge';
export * from './openidRefreshFlight';
+export * from './resourceLock';
export * from './convo';
export * from './chatProject';
export * from './session';
diff --git a/packages/data-schemas/src/types/resourceLock.ts b/packages/data-schemas/src/types/resourceLock.ts
new file mode 100644
index 0000000000..e7bffb44b2
--- /dev/null
+++ b/packages/data-schemas/src/types/resourceLock.ts
@@ -0,0 +1,20 @@
+import type { Document } from 'mongoose';
+
+export interface IResourceLock extends Document {
+ key: string;
+ ownerId: string;
+ expiresAt: Date;
+}
+
+export interface ResourceLockAcquireData {
+ key: string;
+ ownerId: string;
+ ttlMs: number;
+ /** Longest wait for the lock index to be ready; defaults to a short internal bound. */
+ readyWaitMs?: number;
+}
+
+export interface ResourceLockReleaseData {
+ key: string;
+ ownerId: string;
+}