Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🧹 fix: Run the Full Erasure Cascade When an Admin Deletes a User (#17050)

Marco Beretta20d2c0d43d7f
diff --git a/api/server/controllers/UserController.js b/api/server/controllers/UserController.js
index 3f4bdb66b6..70aa6c2db4 100644
--- a/api/server/controllers/UserController.js
+++ b/api/server/controllers/UserController.js
@@ -15,8 +15,7 @@ const {
getWebSearchInstallEntries,
getWebSearchUninstallFields,
openCheckpointDeletion,
- isStopConfirmed,
- waitForGenerationPersistence,
+ stopUserGenerations,
deleteAllSharedLinksWithCleanup,
eraseUserData,
deleteUserMCPServers,
@@ -410,39 +409,7 @@ const deleteUserController = async (req, res) => {
if (!(await quiesceUserSchedules(user.id, scheduleSuspensionToken))) {
throw new Error('Scheduled executions could not be confirmed stopped');
}
- const activeAgentRuns = await GenerationJobManager.getAccountCleanupJobIdsForUser(
- user.id,
- tenantId,
- );
- const activeAgentJobs = await Promise.all(
- activeAgentRuns.map(async (streamId) => ({
- streamId,
- job: await GenerationJobManager.getCleanupJob(streamId),
- })),
- );
- const ownedAgentJobs = activeAgentJobs.filter(
- ({ job }) =>
- job?.metadata?.userId === user.id &&
- (!job.metadata.tenantId || job.metadata.tenantId === tenantId),
- );
- const stopResults = await Promise.all(
- ownedAgentJobs.map(({ streamId, job }) =>
- GenerationJobManager.abortJob(streamId, {
- expectedCreatedAt: job.createdAt,
- awaitProviderDrain: true,
- }),
- ),
- );
- if (stopResults.some((result) => !isStopConfirmed(result))) {
- throw new Error('Agent generations could not be confirmed stopped');
- }
- await Promise.all(
- ownedAgentJobs.map(({ streamId, job }) =>
- waitForGenerationPersistence(streamId, job.createdAt, (id) =>
- GenerationJobManager.getCleanupJob(id),
- ),
- ),
- );
+ await stopUserGenerations(user.id, tenantId, GenerationJobManager);
const appConfig =
req.config ??
diff --git a/api/server/controllers/__tests__/deleteUser.spec.js b/api/server/controllers/__tests__/deleteUser.spec.js
index cbe5dd58ae..13dafb794b 100644
--- a/api/server/controllers/__tests__/deleteUser.spec.js
+++ b/api/server/controllers/__tests__/deleteUser.spec.js
@@ -77,13 +77,9 @@ jest.mock('@librechat/api', () => ({
abortJob: (...args) => mockAbortJob(...args),
},
getOwnedAgentCheckpointScope: jest.fn(() => undefined),
- isStopConfirmed: jest.fn(
- (result) => result?.success === true || result?.failureReason === 'already_settled',
- ),
deleteOwnedAgentCheckpoints: jest.fn(),
- waitForGenerationPersistence: jest.requireActual(
- '../../../../packages/api/src/stream/persistence.ts',
- ).waitForGenerationPersistence,
+ stopUserGenerations: jest.requireActual('../../../../packages/api/src/user/generations.ts')
+ .stopUserGenerations,
openCheckpointDeletion: jest.fn(async (userId, tenantId, _root, cfg) => ({
remember: jest.fn(),
cleanup: () =>
diff --git a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js
index 1d0790a95a..8b9e2196e1 100644
--- a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js
+++ b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js
@@ -1100,7 +1100,12 @@ describe('ResumableAgentController resume metadata', () => {
});
it('rejects any session fenced after authentication before generation execution starts', async () => {
- mockIsAgentTriggerPrincipalActive.mockResolvedValue(false);
+ mockGenerationJobManager.createJob.mockRejectedValueOnce(
+ Object.assign(new Error('Account deletion is in progress'), {
+ code: 'ACCOUNT_DELETION_IN_PROGRESS',
+ status: 409,
+ }),
+ );
const req = {
user: { id: 'user-123' },
body: {
@@ -1118,10 +1123,6 @@ describe('ResumableAgentController resume metadata', () => {
await AgentController(req, res, jest.fn(), initializeClient, null);
expect(mockGenerationJobManager.createJob).toHaveBeenCalledTimes(1);
- expect(mockIsAgentTriggerPrincipalActive).toHaveBeenCalledWith('user-123');
- expect(mockGenerationJobManager.createJob.mock.invocationCallOrder[0]).toBeLessThan(
- mockIsAgentTriggerPrincipalActive.mock.invocationCallOrder[0],
- );
expect(mockGenerationJobManager.beginProviderExecution).not.toHaveBeenCalled();
expect(res.status).toHaveBeenCalledWith(409);
expect(res.json).toHaveBeenCalledWith({
@@ -1132,15 +1133,8 @@ describe('ResumableAgentController resume metadata', () => {
});
expect(initializeClient).not.toHaveBeenCalled();
expect(mockAcceptAgentStartupTelemetry).not.toHaveBeenCalled();
- expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
- 'conversation-123',
- JSON.stringify({
- status: 409,
- code: 'ACCOUNT_DELETION_IN_PROGRESS',
- error: 'Account deletion is in progress',
- }),
- 1000,
- );
+ /** createJob retired the refused job itself, so the controller has none to complete. */
+ expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
expect(mockGenerationJobManager.releaseGeneration).toHaveBeenCalledWith(
'user-123',
'trigger-request',
@@ -1239,10 +1233,6 @@ describe('ResumableAgentController resume metadata', () => {
await AgentController(req, res, jest.fn(), initializeClient, null);
- expect(mockIsAgentTriggerPrincipalActive).toHaveBeenCalledWith('user-123');
- expect(mockIsAgentTriggerPrincipalActive.mock.invocationCallOrder[0]).toBeLessThan(
- mockGenerationJobManager.beginProviderExecution.mock.invocationCallOrder[0],
- );
expect(initializeClient).not.toHaveBeenCalled();
expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
'conversation-123',
diff --git a/api/server/controllers/agents/request.js b/api/server/controllers/agents/request.js
index 51fa58b533..f6cc1694ee 100644
--- a/api/server/controllers/agents/request.js
+++ b/api/server/controllers/agents/request.js
@@ -100,7 +100,6 @@ const {
claimAgentEventActorSuspension,
settleAgentEventActorSuspension,
stampConvoLastResponse,
- isAgentTriggerPrincipalActive,
isSubagentOwnerAdmissible,
appendConvoMessageReference,
} = require('~/models');
@@ -1755,18 +1754,8 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
req.turnStartedAt = jobCreatedAt;
providerExecutionId = job.metadata?.providerExecutionId;
- /** Authentication can precede a slow admission path. Recheck the durable
- * account-deletion fence after the job is committed but before execution
- * starts. This ordering closes both sides of the race for ordinary and
- * trigger-scoped sessions: a fence that wins first rejects this run; a
- * fence that starts after this read must observe the already-created job
- * in account deletion's active-generation drain. */
- if (!(await isAgentTriggerPrincipalActive(userId))) {
- throw Object.assign(new Error('Account deletion is in progress'), {
- code: 'ACCOUNT_DELETION_IN_PROGRESS',
- status: 409,
- });
- }
+ /** Account-deletion admission is enforced inside `createJob`, after the job is committed, so a
+ * fence that wins first refuses this run and one that starts later finds the job in its drain. */
if (req._agentEventBindingParentConversationId != null) {
/** The generation job is the durable marker that a deletion on another replica
* can abort. Recheck only after that marker exists: either the deletion fence
diff --git a/api/server/experimental.js b/api/server/experimental.js
index c444acd04a..1e168f404d 100644
--- a/api/server/experimental.js
+++ b/api/server/experimental.js
@@ -367,6 +367,7 @@ if (cluster.isMaster) {
// but an already-fired scheduled generation can still reach HITL here. Settle
// its durable run when the generic approval runtime expires it.
GenerationJobManager.setApprovalExpiredHandler(recordExpiredScheduleApproval);
+ GenerationJobManager.setAdmissionGuard(agentEventMethods.isAgentTriggerPrincipalActive);
GenerationJobManager.setTerminalHostActionHandler(
createAgentEventTerminalHandler(agentEventMethods, {
resumeDetachedAction: resumeAgentEventDetachedAction,
diff --git a/api/server/index.js b/api/server/index.js
index 469f2863e6..f98c09dfd2 100644
--- a/api/server/index.js
+++ b/api/server/index.js
@@ -139,6 +139,7 @@ const configureGenerationStreams = () => {
cleanupOnComplete: !isEnabled(process.env.STREAM_KEEP_COMPLETED_JOBS),
});
GenerationJobManager.setApprovalExpiredHandler(recordExpiredScheduleApproval);
+ GenerationJobManager.setAdmissionGuard(agentEventMethods.isAgentTriggerPrincipalActive);
GenerationJobManager.setTerminalHostActionHandler(
createAgentEventTerminalHandler(agentEventMethods, {
resumeDetachedAction: resumeAgentEventDetachedAction,
diff --git a/api/server/routes/admin/users.js b/api/server/routes/admin/users.js
index 2823abd9f5..efe861248f 100644
--- a/api/server/routes/admin/users.js
+++ b/api/server/routes/admin/users.js
@@ -1,7 +1,14 @@
const express = require('express');
const mongoose = require('mongoose');
-const { createAdminUsersHandlers, revokeUserCodeEnvironmentWorkers } = require('@librechat/api');
-const { SystemCapabilities } = require('@librechat/data-schemas');
+const {
+ GenerationJobManager,
+ openCheckpointDeletion,
+ deleteAllSharedLinksWithCleanup,
+ createErasureResolver,
+ createAdminUsersHandlers,
+ revokeUserCodeEnvironmentWorkers,
+} = require('@librechat/api');
+const { SystemCapabilities, runAsSystem, tenantStorage } = require('@librechat/data-schemas');
const { requireCapability } = require('~/server/middleware/roles/capabilities');
const { requireJwtAuth } = require('~/server/middleware');
const {
@@ -10,11 +17,37 @@ const {
cancelAgentTriggerUserPurge,
purgeAgentTriggerDeliveriesForUser,
} = require('~/server/services/Agents/triggers');
+const { processDeleteRequest } = require('~/server/services/Files/process');
+const { invalidateCachedTools } = require('~/server/services/Config/getCachedTools');
+const { getMCPManager, getMCPServersRegistry } = require('~/config');
+const { maybeUninstallOAuthMCP } = require('~/server/services/MCP/oauthCleanup');
+const subagentThreadTaskStore = require('~/server/services/Endpoints/agents/subagentThreadStore');
+const {
+ quiesceUserSchedules,
+ restoreUserSchedulesFromDeletion,
+} = require('~/server/services/Schedules');
const db = require('~/models');
const { getAppConfig, invalidateCodeEnvironmentConfigCache } = require('~/server/services/Config');
const router = express.Router();
+const resolveErasure = createErasureResolver({
+ withTenant: (tenantId, operation) =>
+ tenantId ? tenantStorage.run({ tenantId }, operation) : runAsSystem(operation),
+ db,
+ getAppConfig,
+ getMCPManager,
+ processDeleteRequest,
+ invalidateCachedTools,
+ mcpOAuth: {
+ getRegistry: getMCPServersRegistry,
+ getUserPrincipals: db.getUserPrincipals,
+ uninstallOAuthMCP: maybeUninstallOAuthMCP,
+ },
+ openCheckpointDeletion,
+ deleteSharedLinks: (userId) => deleteAllSharedLinksWithCleanup(userId, { awaitCleanup: true }),
+});
+
const requireAdminAccess = requireCapability(SystemCapabilities.ACCESS_ADMIN);
const requireReadUsers = requireCapability(SystemCapabilities.READ_USERS);
// const requireManageUsers = requireCapability(SystemCapabilities.MANAGE_USERS);
@@ -40,7 +73,12 @@ const handlers = createAdminUsersHandlers({
deleteConfig: db.deleteConfig,
deleteAclEntries: db.deleteAclEntries,
deletePasskeysByUser: db.deletePasskeysByUser,
- deleteTokens: db.deleteTokens,
+ resolveErasure,
+ cancelSubagentTasks: (userId, tenantId) =>
+ subagentThreadTaskStore.cancelAndDrainForOwner(userId, tenantId),
+ quiesceUserSchedules,
+ restoreUserSchedulesFromDeletion,
+ generationJobs: GenerationJobManager,
});
router.use(requireJwtAuth, requireAdminAccess);
diff --git a/packages/api/src/admin/users.spec.ts b/packages/api/src/admin/users.spec.ts
index 1b969cfef6..3c1aea1303 100644
--- a/packages/api/src/admin/users.spec.ts
+++ b/packages/api/src/admin/users.spec.ts
@@ -5,6 +5,11 @@ import type { Response } from 'express';
import type { ServerRequest } from '~/types/http';
import type { AdminUsersDeps } from './users';
import { createAdminUsersHandlers } from './users';
+import { eraseUserData } from '~/user/erasure';
+
+jest.mock('~/user/erasure', () => ({
+ eraseUserData: jest.fn().mockResolvedValue(undefined),
+}));
jest.mock('@librechat/data-schemas', () => ({
...jest.requireActual('@librechat/data-schemas'),
@@ -68,12 +73,24 @@ function createDeps(overrides: Partial<AdminUsersDeps> = {}): AdminUsersDeps {
deleteConfig: jest.fn().mockResolvedValue(null),
deleteAclEntries: jest.fn().mockResolvedValue(undefined),
deletePasskeysByUser: jest.fn().mockResolvedValue(0),
- deleteTokens: jest.fn().mockResolvedValue({ deletedCount: 0 }),
+ resolveErasure: jest.fn().mockResolvedValue({ deps: {} }),
+ cancelSubagentTasks: jest.fn().mockResolvedValue(undefined),
+ quiesceUserSchedules: jest.fn().mockResolvedValue(true),
+ restoreUserSchedulesFromDeletion: jest.fn().mockResolvedValue(undefined),
+ generationJobs: {
+ getAccountCleanupJobIdsForUser: jest.fn().mockResolvedValue([]),
+ getCleanupJob: jest.fn().mockResolvedValue(undefined),
+ abortJob: jest.fn().mockResolvedValue(null),
+ },
...overrides,
};
}
describe('createAdminUsersHandlers', () => {
+ beforeEach(() => {
+ jest.mocked(eraseUserData).mockClear();
+ });
+
describe('listUsers', () => {
it('returns paginated users with total count', async () => {
const users = [
@@ -462,32 +479,118 @@ describe('createAdminUsersHandlers', () => {
expect(deps.deletePasskeysByUser).toHaveBeenCalledWith(validUserId);
});
- it("removes the user's stored credentials before the user", async () => {
- const deps = createDeps();
+ it('runs the shared erasure cascade for the target before removing the user', async () => {
+ const checkpointer = { ttl: 60 };
+ const erasureDeps = { marker: true };
+ const deps = createDeps({
+ findUsers: jest.fn().mockResolvedValue([mockUser({ role: 'USER', tenantId: 'tenant-1' })]),
+ resolveErasure: jest.fn().mockResolvedValue({ deps: erasureDeps, checkpointer }),
+ });
const handlers = createAdminUsersHandlers(deps);
const { req, res, status } = createReqRes({ params: { id: validUserId } });
await handlers.deleteUser(req, res);
expect(status).toHaveBeenCalledWith(200);
- expect(deps.deleteTokens).toHaveBeenCalledWith({ userId: validUserId });
- expect(jest.mocked(deps.deleteTokens).mock.invocationCallOrder[0]).toBeLessThan(
+ expect(deps.resolveErasure).toHaveBeenCalledWith(
+ expect.objectContaining({
+ id: validUserId,
+ role: 'USER',
+ tenantId: 'tenant-1',
+ idOnTheSource: null,
+ profile: expect.objectContaining({ role: 'USER' }),
+ }),
+ );
+ expect(eraseUserData).toHaveBeenCalledWith(
+ { userId: validUserId, tenantId: 'tenant-1', checkpointer },
+ erasureDeps,
+ );
+ expect(jest.mocked(eraseUserData).mock.invocationCallOrder[0]).toBeLessThan(
jest.mocked(deps.deleteUserById).mock.invocationCallOrder[0],
);
});
- it('keeps the user, and releases the fence, when its credentials cannot be removed', async () => {
+ it('keeps the user, and releases the fence, when the erasure cascade fails', async () => {
+ jest.mocked(eraseUserData).mockRejectedValueOnce(new Error('storage down'));
+ const deps = createDeps();
+ const handlers = createAdminUsersHandlers(deps);
+ const { req, res, status, json } = createReqRes({ params: { id: validUserId } });
+
+ await handlers.deleteUser(req, res);
+
+ expect(status).toHaveBeenCalledWith(500);
+ expect(json).toHaveBeenCalledWith({ error: 'Failed to delete user' });
+ expect(deps.deleteUserById).not.toHaveBeenCalled();
+ expect(deps.cancelAgentTriggerUserDeletion).toHaveBeenCalled();
+ expect(deps.cancelAgentTriggerUserPurge).toHaveBeenCalled();
+ });
+
+ it('suspends schedules before erasing and restores them when the deletion fails', async () => {
+ jest.mocked(eraseUserData).mockRejectedValueOnce(new Error('storage down'));
+ const deps = createDeps();
+ const handlers = createAdminUsersHandlers(deps);
+ const { req, res, status } = createReqRes({ params: { id: validUserId } });
+
+ await handlers.deleteUser(req, res);
+
+ const token = jest.mocked(deps.quiesceUserSchedules).mock.calls[0][1];
+ expect(status).toHaveBeenCalledWith(500);
+ expect(deps.restoreUserSchedulesFromDeletion).toHaveBeenCalledWith(validUserId, token);
+ expect(
+ jest.mocked(deps.restoreUserSchedulesFromDeletion).mock.invocationCallOrder[0],
+ ).toBeLessThan(jest.mocked(deps.cancelAgentTriggerUserDeletion).mock.invocationCallOrder[0]);
+ });
+
+ it('takes the trigger deletion fence before listing generations to stop', async () => {
+ const deps = createDeps();
+ const handlers = createAdminUsersHandlers(deps);
+ const { req, res } = createReqRes({ params: { id: validUserId } });
+
+ await handlers.deleteUser(req, res);
+
+ expect(
+ jest.mocked(deps.beginAgentTriggerUserDeletion).mock.invocationCallOrder[0],
+ ).toBeLessThan(
+ jest.mocked(deps.generationJobs.getAccountCleanupJobIdsForUser).mock.invocationCallOrder[0],
+ );
+ });
+
+ it('does not erase when schedules cannot be confirmed stopped', async () => {
+ const deps = createDeps({ quiesceUserSchedules: jest.fn().mockResolvedValue(false) });
+ const handlers = createAdminUsersHandlers(deps);
+ const { req, res, status } = createReqRes({ params: { id: validUserId } });
+
+ await handlers.deleteUser(req, res);
+
+ expect(status).toHaveBeenCalledWith(500);
+ expect(eraseUserData).not.toHaveBeenCalled();
+ expect(deps.deleteUserById).not.toHaveBeenCalled();
+ });
+
+ it('aborts the cleanup jobs of the target and refuses to erase when a stop is unconfirmed', async () => {
+ const abortJob = jest.fn().mockResolvedValue(null);
const deps = createDeps({
- deleteTokens: jest.fn().mockRejectedValue(new Error('mongo down')),
+ generationJobs: {
+ getAccountCleanupJobIdsForUser: jest.fn().mockResolvedValue(['s1', 's2']),
+ getCleanupJob: jest.fn(async (id: string) => ({
+ createdAt: 5,
+ metadata: { userId: id === 's1' ? validUserId : 'someone-else' },
+ })),
+ abortJob,
+ } as unknown as AdminUsersDeps['generationJobs'],
});
const handlers = createAdminUsersHandlers(deps);
const { req, res, status } = createReqRes({ params: { id: validUserId } });
await handlers.deleteUser(req, res);
+ expect(abortJob).toHaveBeenCalledTimes(1);
+ expect(abortJob).toHaveBeenCalledWith('s1', {
+ expectedCreatedAt: 5,
+ awaitProviderDrain: true,
+ });
expect(status).toHaveBeenCalledWith(500);
- expect(deps.deleteUserById).not.toHaveBeenCalled();
- expect(deps.cancelAgentTriggerUserDeletion).toHaveBeenCalled();
+ expect(eraseUserData).not.toHaveBeenCalled();
});
it('returns success even when cascade cleanup partially fails', async () => {
diff --git a/packages/api/src/admin/users.ts b/packages/api/src/admin/users.ts
index e6a1e5f431..ae8ac19c9c 100644
--- a/packages/api/src/admin/users.ts
+++ b/packages/api/src/admin/users.ts
@@ -1,4 +1,5 @@
import { Types } from 'mongoose';
+import { randomUUID } from 'node:crypto';
import { PrincipalType, SystemRoles } from 'librechat-data-provider';
import { logger, isValidObjectIdString } from '@librechat/data-schemas';
import type {
@@ -8,15 +9,61 @@ import type {
AdminUserSearchResult,
UserDeleteResult,
} from '@librechat/data-schemas';
+import type { TCheckpointerConfig } from 'librechat-data-provider';
import type { FilterQuery } from 'mongoose';
import type { Response } from 'express';
+import type { UserGenerationControl } from '~/user/generations';
+import type { UserErasureDeps } from '~/user/erasure';
import type { ServerRequest } from '~/types/http';
+import { stopUserGenerations } from '~/user/generations';
import { parsePagination } from './pagination';
+import { eraseUserData } from '~/user/erasure';
const MAX_SEARCH_LENGTH = 200;
+const ERASURE_PROFILE_KEYS = [
+ 'name',
+ 'username',
+ 'email',
+ 'provider',
+ 'googleId',
+ 'facebookId',
+ 'openidId',
+ 'samlId',
+ 'ldapId',
+ 'githubId',
+ 'discordId',
+ 'appleId',
+ 'emailVerified',
+ 'twoFactorEnabled',
+ 'termsAccepted',
+ 'termsAcceptedAt',
+] as const;
+
+const ERASURE_TARGET_FIELDS = ['role', 'tenantId', 'idOnTheSource', ...ERASURE_PROFILE_KEYS].join(
+ ' ',
+);
+
+/** The target's plain profile fields, which storage header templates such as the user email read. */
+export type ErasureTargetProfile = Partial<Pick<IUser, (typeof ERASURE_PROFILE_KEYS)[number]>>;
+
const USER_LIST_FIELDS = '_id name username email avatar role provider createdAt updatedAt';
+/** The target account's identity, from which the caller builds storage and MCP cleanup. */
+export interface AdminErasureTarget {
+ id: string;
+ role?: string;
+ tenantId?: string;
+ /** External identity key, so principal resolution reuses the loaded user instead of re-reading it. */
+ idOnTheSource?: string | null;
+ profile?: ErasureTargetProfile;
+}
+
+export interface AdminErasureContext {
+ deps: UserErasureDeps;
+ checkpointer?: TCheckpointerConfig;
+}
+
export interface AdminUsersDeps {
findUsers: (
searchCriteria: FilterQuery<IUser>,
@@ -38,14 +85,22 @@ export interface AdminUsersDeps {
cancelAgentTriggerUserPurge: (userId: string, fenceStartedAt: Date) => Promise<boolean>;
purgeAgentTriggerDeliveriesForUser: (userId: string) => Promise<void>;
revokeUserCodeEnvironmentWorkers?: (userId: string) => Promise<number>;
+ /** Cancels and drains the account's subagent thread tasks. */
+ cancelSubagentTasks?: (userId: string, tenantId?: string) => Promise<unknown>;
/**
- * Thin data-layer delete — removes the User document only.
- * Full cascade of user-owned resources (conversations, messages, files, tokens, etc.)
- * is handled by `UserController.deleteUserController` in the self-delete flow.
- * This admin endpoint fences durable triggers around the user commit and currently
- * cascades Config and AclEntries.
- * A future iteration should consolidate the full cascade into a shared service function.
+ * Reversibly suspends the account's schedules under a per-attempt token; false means the
+ * executions could not be confirmed stopped. Restored if the deletion fails.
*/
+ quiesceUserSchedules: (userId: string, token: string) => Promise<boolean>;
+ restoreUserSchedulesFromDeletion: (userId: string, token: string) => Promise<unknown>;
+ generationJobs: UserGenerationControl;
+ /**
+ * Resolves the dependencies of the shared `eraseUserData` cascade for the account being
+ * deleted, the same set the self-service controller and the delete-user CLI pass. Storage
+ * cleanup must run as the target user, not the calling admin.
+ */
+ resolveErasure: (target: AdminErasureTarget) => Promise<AdminErasureContext>;
+ /** Thin data-layer delete: removes the User document after the erasure cascade has run. */
deleteUserById: (userId: string) => Promise<UserDeleteResult>;
deleteUserCodeEnvironments: (userId: string | Types.ObjectId) => Promise<number>;
invalidateCodeEnvironmentConfigCache: (tenantId?: string) => Promise<void>;
@@ -62,12 +117,6 @@ export interface AdminUsersDeps {
* keep authentication material and can collide with a later registration.
*/
deletePasskeysByUser: (userId: string) => Promise<number>;
- /**
- * Stored credentials the user holds, such as a connected account's OAuth grant. Removed before
- * the user is, as the self-delete flow does, so a failure leaves the account to retry rather
- * than its credentials behind.
- */
- deleteTokens: (query: { userId: string }) => Promise<unknown>;
}
export function createAdminUsersHandlers(deps: AdminUsersDeps): {
@@ -91,7 +140,11 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {
deleteConfig,
deleteAclEntries,
deletePasskeysByUser,
- deleteTokens,
+ resolveErasure,
+ cancelSubagentTasks,
+ quiesceUserSchedules,
+ restoreUserSchedulesFromDeletion,
+ generationJobs,
} = deps;
async function listUsersHandler(req: ServerRequest, res: Response) {
@@ -173,6 +226,7 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {
async function deleteUserHandler(req: ServerRequest, res: Response) {
let targetUserId: string | undefined;
let triggerDeletionFence: Date | undefined;
+ let scheduleSuspensionToken: string | undefined;
let userDeleted = false;
try {
@@ -188,7 +242,7 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {
return res.status(403).json({ error: 'Cannot delete your own account' });
}
- const [targetUser] = await findUsers({ _id: id }, 'role tenantId', { limit: 1 });
+ const [targetUser] = await findUsers({ _id: id }, ERASURE_TARGET_FIELDS, { limit: 1 });
if (targetUser?.role === SystemRoles.ADMIN) {
const adminCount = await countUsers({ role: SystemRoles.ADMIN });
if (adminCount <= 1) {
@@ -209,7 +263,24 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {
await prepareAgentTriggerUserPurge(id, triggerDeletionFence, targetUser?.tenantId);
await drainAgentTriggerDeliveriesForUser(id);
- await deleteTokens({ userId: id });
+ await cancelSubagentTasks?.(id, targetUser?.tenantId);
+ scheduleSuspensionToken = randomUUID();
+ if (!(await quiesceUserSchedules(id, scheduleSuspensionToken))) {
+ throw new Error('Scheduled executions could not be confirmed stopped');
+ }
+ await stopUserGenerations(id, targetUser?.tenantId, generationJobs);
+
+ const { deps: erasureDeps, checkpointer } = await resolveErasure({
+ id,
+ role: targetUser?.role,
+ tenantId: targetUser?.tenantId,
+ idOnTheSource: targetUser?.idOnTheSource ?? null,
+ profile: targetUser,
+ });
+ await eraseUserData(
+ { userId: id, tenantId: targetUser?.tenantId, checkpointer },
+ erasureDeps,
+ );
const result = await deleteUserById(id);
if (result.deletedCount === 0) {
@@ -256,6 +327,17 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {
return res.status(200).json({ message: result.message || 'User deleted successfully' });
} catch (error) {
+ /** Restore before releasing the fence, as the self-service flow does. */
+ if (targetUserId != null && scheduleSuspensionToken != null && !userDeleted) {
+ try {
+ await restoreUserSchedulesFromDeletion(targetUserId, scheduleSuspensionToken);
+ } catch (restoreError) {
+ logger.error(
+ `[adminUsers] failed to restore suspended schedules for user ${targetUserId} under token ${scheduleSuspensionToken}:`,
+ restoreError,
+ );
+ }
+ }
if (targetUserId != null && triggerDeletionFence != null && !userDeleted) {
try {
await cancelAgentTriggerUserPurge(targetUserId, triggerDeletionFence);
diff --git a/packages/api/src/agents/remote/lifecycle.spec.ts b/packages/api/src/agents/remote/lifecycle.spec.ts
index ea8400d976..cad6969e8e 100644
--- a/packages/api/src/agents/remote/lifecycle.spec.ts
+++ b/packages/api/src/agents/remote/lifecycle.spec.ts
@@ -87,13 +87,10 @@ describe('Agent execution enrollment', () => {
expect(beginProviderExecution).not.toHaveBeenCalled();
await expect(manager.getCleanupBlockingJobIdsForUser('user-1')).resolves.toEqual([]);
- await expect(manager.getJobStore().getJob('chatcmpl-deleting')).resolves.toMatchObject({
- status: 'error',
- providerDrained: true,
- });
+ await expect(manager.getJobStore().getJob('chatcmpl-deleting')).resolves.toBeNull();
});
- it('preserves principal-check infrastructure failures after retiring the run', async () => {
+ it('reports principal-check infrastructure failures with a sanitized error after retiring the run', async () => {
const infrastructureError = new Error('principal store unavailable');
await expect(
@@ -104,7 +101,7 @@ describe('Agent execution enrollment', () => {
},
{ manager },
),
- ).rejects.toBe(infrastructureError);
+ ).rejects.toMatchObject({ code: 'GENERATION_ADMISSION_UNAVAILABLE' });
await expect(manager.getCleanupBlockingJobIdsForUser('user-1')).resolves.toEqual([]);
});
diff --git a/packages/api/src/agents/remote/lifecycle.ts b/packages/api/src/agents/remote/lifecycle.ts
index b80aabbdb4..3191c75335 100644
--- a/packages/api/src/agents/remote/lifecycle.ts
+++ b/packages/api/src/agents/remote/lifecycle.ts
@@ -1,6 +1,6 @@
import type { GenerationJobManagerClass } from '~/stream';
import type { GenerationJob } from '~/types/stream';
-import { GenerationJobManager } from '~/stream';
+import { GenerationJobManager, GenerationAdmissionDeniedError } from '~/stream';
const ACCOUNT_DELETION_ERROR = 'Account deletion is in progress';
const REMOTE_EXECUTION_ERROR = 'Remote agent execution failed';
@@ -190,38 +190,33 @@ export class AgentExecutionEnrollment {
}
}
-async function retireRejectedEnrollment(
- manager: GenerationJobManagerClass,
- job: GenerationJob,
-): Promise<void> {
- await manager.completeJob(job.streamId, ACCOUNT_DELETION_ERROR, job.createdAt);
-}
-
export async function enrollAgentExecution(
params: AgentExecutionEnrollmentParams,
deps: AgentExecutionEnrollmentDeps = { manager: GenerationJobManager },
): Promise<AgentExecutionEnrollment> {
const { runId, userId, conversationId, agentId, protocol, isPrincipalActive } = params;
- const job = await deps.manager.createJob(runId, userId, conversationId, {
- initialMetadata: {
- agent_id: agentId,
- endpoint: protocol,
- model: agentId,
- responseMessageId: runId,
- },
- });
-
- let active = false;
+ let job: GenerationJob;
try {
- active = await isPrincipalActive(userId);
+ /** The principal predicate replaces the default guard, so admission is one post-commit read
+ * and a refused run takes the manager's retire-and-purge path. */
+ job = await deps.manager.createJob(runId, userId, conversationId, {
+ admissionCheck: isPrincipalActive,
+ initialMetadata: {
+ agent_id: agentId,
+ endpoint: protocol,
+ model: agentId,
+ responseMessageId: runId,
+ },
+ });
} catch (error) {
- await retireRejectedEnrollment(deps.manager, job).catch(() => undefined);
+ if (error instanceof GenerationAdmissionDeniedError) {
+ throw new AgentExecutionAdmissionError(
+ ACCOUNT_DELETION_ERROR,
+ 'ACCOUNT_DELETION_IN_PROGRESS',
+ );
+ }
throw error;
}
- if (!active) {
- await retireRejectedEnrollment(deps.manager, job);
- throw new AgentExecutionAdmissionError(ACCOUNT_DELETION_ERROR, 'ACCOUNT_DELETION_IN_PROGRESS');
- }
return new AgentExecutionEnrollment(deps.manager, job);
}
diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts
index 9f721c2a3d..48353dbdce 100644
--- a/packages/api/src/stream/GenerationJobManager.ts
+++ b/packages/api/src/stream/GenerationJobManager.ts
@@ -44,28 +44,31 @@ import type { RecoveredSteerPayload } from './SteerRecovery';
import type { SteerContentView } from './SteeringLifecycle';
import type { GenerationJobStore } from '~/app/metrics';
import type * as t from '~/types';
-import {
- recordGenerationStreamEarlyBufferOverflow,
- recordGenerationStreamResumePendingEvents,
- recordGenerationStreamSubscription,
- recordGenerationStreamAttachment,
- recordGenerationStreamRecovery,
- setGenerationJobsInFlight,
- recordGenerationJob,
-} from '~/app/metrics';
import {
GenerationPublicationFencedError,
JobCreationSupersededError,
JobPredecessorMismatchError,
+ GenerationAdmissionDeniedError,
+ GenerationAdmissionUnavailableError,
isPendingActionStale,
isPendingActionExpired,
PAUSE_PERSISTENCE_TIMEOUT_ERROR,
PROVIDER_DRAIN_TIMEOUT_MS,
STEER_QUEUE_MAX_DEPTH,
} from './interfaces/IJobStore';
+import {
+ recordGenerationStreamEarlyBufferOverflow,
+ recordGenerationStreamResumePendingEvents,
+ recordGenerationStreamSubscription,
+ recordGenerationStreamAttachment,
+ recordGenerationStreamRecovery,
+ setGenerationJobsInFlight,
+ recordGenerationJob,
+} from '~/app/metrics';
import { isRecoveredSteerPayload, RecoveredSteerPayloadMismatchError } from './SteerRecovery';
import { projectTerminalEvent } from './terminalProjection';
import { assertJobStoreV2 } from './jobStoreCapabilities';
+import { getSafeErrorMetadata } from '~/utils/errors';
/**
* Tombstone budget per generation. Twice the queue depth so a full queue's
@@ -143,6 +146,7 @@ const IDEMPOTENCY_TTL_SECONDS = 25 * 60 * 60;
const LEGACY_IDEMPOTENCY_TTL_SECONDS = 26 * 60 * 60;
const OAUTH_TOOL_CALL_PREFIX = `oauth${Constants.mcp_delimiter}`;
const SHUTDOWN_SUBSCRIBER_ERROR = 'Server is shutting down';
+const ACCOUNT_DELETION_JOB_ERROR = 'Generation stopped because the account is being deleted';
const SHUTDOWN_JOB_ERROR = 'Generation interrupted because its server shut down';
const SHUTTING_DOWN_ERROR = 'Generation job manager is shutting down';
/** Internal transport signal: durable terminal state exists, but its DONE
@@ -662,6 +666,8 @@ export type TerminalHostActionHandler = (
) => void | Promise<void>;
export interface CreateGenerationJobOptions {
+ /** Replaces the installed admission guard for this job, for callers with a stricter predicate. */
+ admissionCheck?: (userId: string) => Promise<boolean>;
startupTelemetry?: AgentStartupTelemetry;
initialMetadata?: Partial<t.GenerationJobMetadata>;
/** A terminal steer being handed off as this new normal turn. The store
@@ -886,6 +892,8 @@ type DeferredDelivery =
* ```
*/
class GenerationJobManagerClass {
+ private admissionGuard?: (userId: string) => Promise<boolean>;
+
/** Job metadata + content state storage - swappable for Redis, etc. */
private jobStore: IJobStoreV2;
/** Guarded human-review lifecycle (pause / resolve / expire) over the store. */
@@ -1142,6 +1150,38 @@ class GenerationJobManagerClass {
this.approvalExpiredHandler = handler;
}
+ /**
+ * Installs the account admission check, which `createJob` runs before it creates any durable
+ * state. The caller supplies it, keeping the stream package storage-free.
+ */
+ setAdmissionGuard(guard?: (userId: string) => Promise<boolean>): void {
+ this.admissionGuard = guard;
+ }
+
+ private async assertAdmitted(
+ streamId: string,
+ userId: string,
+ options: CreateGenerationJobOptions,
+ ): Promise<void> {
+ const admissionCheck = options.admissionCheck ?? this.admissionGuard;
+ if (admissionCheck == null) {
+ return;
+ }
+ let admitted: boolean;
+ try {
+ admitted = await admissionCheck(userId);
+ } catch (error) {
+ logger.error(
+ `[GenerationJobManager] Admission check failed for ${streamId}:`,
+ getSafeErrorMetadata(error),
+ );
+ throw new GenerationAdmissionUnavailableError();
+ }
+ if (!admitted) {
+ throw new GenerationAdmissionDeniedError();
+ }
+ }
+
/** Installs a durable, generation-fenced terminal lifecycle adapter. */
setTerminalHostActionHandler(handler?: TerminalHostActionHandler): void {
this.terminalHostActionHandler = handler;
@@ -2540,6 +2580,9 @@ class GenerationJobManagerClass {
throw new Error('Invalid active generation predecessor policy');
}
+ /** Refuse before any durable state exists; the common case leaves nothing to clean up. */
+ await this.assertAdmitted(streamId, userId, options);
+
const tenantId = getTenantId();
const safeTenantId = tenantId && tenantId !== SYSTEM_TENANT_ID ? tenantId : undefined;
const creationAttemptId = randomUUID();
@@ -2731,7 +2774,21 @@ class GenerationJobManagerClass {
}
throw new Error('Generation job was replaced during initialization');
}
- if (this.shuttingDown) {
+ /**
+ * Race-free admission: account deletion sets its marker first and then lists this user's jobs.
+ * A job committed before that listing is stopped by the drain; a job committed after it
+ * checks the marker here, after the marker was set, and retires itself through the normal
+ * terminal path. The marker is therefore always seen by one side or the other.
+ */
+ let postCommitRefusal: Error | undefined;
+ if (!this.shuttingDown) {
+ try {
+ await this.assertAdmitted(streamId, userId, options);
+ } catch (error) {
+ postCommitRefusal = error as Error;
+ }
+ }
+ if (this.shuttingDown || postCommitRefusal != null) {
await this.processDurableReplacementReceipts(streamId, jobData, conversationId).catch(
(handoffError) => {
logger.error(
@@ -2741,20 +2798,35 @@ class GenerationJobManagerClass {
return false;
},
);
- await this.completeJob(streamId, SHUTDOWN_JOB_ERROR, jobData.createdAt);
- if (jobData.providerExecutionId) {
- await this.markProviderExecutionDrained(
+ try {
+ await this.completeJob(
streamId,
+ postCommitRefusal != null ? ACCOUNT_DELETION_JOB_ERROR : SHUTDOWN_JOB_ERROR,
jobData.createdAt,
- jobData.providerExecutionId,
);
+ if (jobData.providerExecutionId) {
+ await this.markProviderExecutionDrained(
+ streamId,
+ jobData.createdAt,
+ jobData.providerExecutionId,
+ );
+ }
+ const replacedLocal = this.runtimeState.get(streamId);
+ if (replacedLocal != null && replacedLocal.createdAt < jobData.createdAt) {
+ replacedLocal.abortController.abort();
+ this.retireStoppedPredecessorRuntime(streamId, jobData.createdAt);
+ }
+ } catch (retireError) {
+ if (postCommitRefusal == null) {
+ throw retireError;
+ }
+ logger.error(
+ `[GenerationJobManager] Failed to retire refused generation ${streamId}:`,
+ getSafeErrorMetadata(retireError),
+ );
+ throw new GenerationAdmissionUnavailableError();
}
- const replacedLocal = this.runtimeState.get(streamId);
- if (replacedLocal != null && replacedLocal.createdAt < jobData.createdAt) {
- replacedLocal.abortController.abort();
- this.retireStoppedPredecessorRuntime(streamId, jobData.createdAt);
- }
- throw new Error(SHUTTING_DOWN_ERROR);
+ throw postCommitRefusal ?? new Error(SHUTTING_DOWN_ERROR);
}
if (!(await this.processDurableReplacementReceipts(streamId, jobData, conversationId))) {
diff --git a/packages/api/src/stream/__tests__/admissionGuard.spec.ts b/packages/api/src/stream/__tests__/admissionGuard.spec.ts
new file mode 100644
index 0000000000..b3a78b85b6
--- /dev/null
+++ b/packages/api/src/stream/__tests__/admissionGuard.spec.ts
@@ -0,0 +1,112 @@
+import {
+ GenerationAdmissionDeniedError,
+ GenerationAdmissionUnavailableError,
+} from '~/stream/interfaces/IJobStore';
+import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport';
+import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore';
+import { GenerationJobManagerClass } from '~/stream/GenerationJobManager';
+import { stopUserGenerations } from '~/user/generations';
+
+describe('GenerationJobManager admission guard', () => {
+ let manager: GenerationJobManagerClass;
+ let store: InMemoryJobStore;
+
+ beforeEach(() => {
+ store = new InMemoryJobStore({ ttlAfterComplete: 60000 });
+ manager = new GenerationJobManagerClass();
+ manager.configure({ jobStore: store, eventTransport: new InMemoryEventTransport() });
+ manager.initialize();
+ });
+
+ afterEach(async () => {
+ await manager.destroy();
+ });
+
+ it('refuses before creating any job state while the account deletion has begun', async () => {
+ manager.setAdmissionGuard(async () => false);
+
+ await expect(manager.createJob('denied', 'user-1', 'denied')).rejects.toBeInstanceOf(
+ GenerationAdmissionDeniedError,
+ );
+ await expect(manager.createJob('denied', 'user-1', 'denied')).rejects.toMatchObject({
+ code: 'ACCOUNT_DELETION_IN_PROGRESS',
+ status: 409,
+ message: 'Account deletion is in progress',
+ });
+ expect(await store.getJob('denied')).toBeNull();
+ });
+
+ it('refuses with a sanitized code when the guard fails and creates nothing', async () => {
+ manager.setAdmissionGuard(async () => {
+ throw new Error('mongodb://internal-host:27017 lookup down');
+ });
+
+ const error = await manager.createJob('failed', 'user-1', 'failed').catch((e) => e);
+
+ expect(error).toBeInstanceOf(GenerationAdmissionUnavailableError);
+ expect(error).toMatchObject({
+ code: 'GENERATION_ADMISSION_UNAVAILABLE',
+ message: 'Generation admission could not be verified',
+ });
+ expect(await store.getJob('failed')).toBeNull();
+ });
+
+ it('uses a per-job admission check in place of the installed guard', async () => {
+ const guard = jest.fn(async () => true);
+ manager.setAdmissionGuard(guard);
+
+ await expect(
+ manager.createJob('own-check', 'user-1', 'own-check', { admissionCheck: async () => false }),
+ ).rejects.toBeInstanceOf(GenerationAdmissionDeniedError);
+ expect(guard).not.toHaveBeenCalled();
+ });
+
+ it('admits jobs while the account is active', async () => {
+ manager.setAdmissionGuard(async () => true);
+
+ await expect(manager.createJob('ok', 'user-1', 'ok')).resolves.toMatchObject({
+ streamId: 'ok',
+ });
+ });
+
+ it('retires a job committed after the marker was set and fails with the stable code', async () => {
+ let marker = false;
+ manager.setAdmissionGuard(async () => !marker);
+ const created = jest.spyOn(store, 'createJob');
+ created.mockImplementationOnce(async (...args) => {
+ marker = true;
+ return InMemoryJobStore.prototype.createJob.apply(store, args);
+ });
+
+ await expect(manager.createJob('late', 'user-1', 'late')).rejects.toBeInstanceOf(
+ GenerationAdmissionDeniedError,
+ );
+ expect((await store.getJob('late'))?.status).toBe('error');
+ });
+
+ it('maps a retirement failure after a refused commit to the sanitized error', async () => {
+ let marker = false;
+ manager.setAdmissionGuard(async () => !marker);
+ jest.spyOn(store, 'createJob').mockImplementationOnce(async (...args) => {
+ marker = true;
+ return InMemoryJobStore.prototype.createJob.apply(store, args);
+ });
+ jest.spyOn(manager, 'completeJob').mockRejectedValueOnce(new Error('redis://internal:6379'));
+
+ await expect(manager.createJob('retire-fail', 'user-1', 'retire-fail')).rejects.toMatchObject({
+ code: 'GENERATION_ADMISSION_UNAVAILABLE',
+ message: 'Generation admission could not be verified',
+ });
+ });
+
+ it('stops a job committed before the deletion listing', async () => {
+ let marker = false;
+ manager.setAdmissionGuard(async () => !marker);
+ await manager.createJob('before', 'user-1', 'before');
+
+ marker = true;
+ await stopUserGenerations('user-1', undefined, manager);
+
+ expect((await store.getJob('before'))?.status).not.toBe('running');
+ });
+});
diff --git a/packages/api/src/stream/index.ts b/packages/api/src/stream/index.ts
index c26c310b7c..b54ffdcb0d 100644
--- a/packages/api/src/stream/index.ts
+++ b/packages/api/src/stream/index.ts
@@ -31,6 +31,8 @@ export type {
export {
GenerationPublicationFencedError,
JobPredecessorMismatchError,
+ GenerationAdmissionDeniedError,
+ GenerationAdmissionUnavailableError,
isPendingActionExpired,
isPendingActionStale,
} from './interfaces/IJobStore';
diff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.ts
index d160933229..f9e040252c 100644
--- a/packages/api/src/stream/interfaces/IJobStore.ts
+++ b/packages/api/src/stream/interfaces/IJobStore.ts
@@ -460,6 +460,28 @@ export class JobPredecessorMismatchError extends Error {
}
}
+/** Thrown by `createJob` when the owner's account deletion began before the job could be admitted. */
+export class GenerationAdmissionDeniedError extends Error {
+ readonly code = 'ACCOUNT_DELETION_IN_PROGRESS';
+ readonly status = 409;
+
+ constructor() {
+ super('Account deletion is in progress');
+ this.name = 'GenerationAdmissionDeniedError';
+ }
+}
+
+/** Thrown by `createJob` when the account admission check itself failed; carries no lookup diagnostics. */
+export class GenerationAdmissionUnavailableError extends Error {
+ readonly code = 'GENERATION_ADMISSION_UNAVAILABLE';
+ readonly status = 503;
+
+ constructor() {
+ super('Generation admission could not be verified');
+ this.name = 'GenerationAdmissionUnavailableError';
+ }
+}
+
export type JobMetadataPatch = Partial<
Pick<
SerializableJobData,
diff --git a/packages/api/src/user/generations.spec.ts b/packages/api/src/user/generations.spec.ts
new file mode 100644
index 0000000000..da2681a3fe
--- /dev/null
+++ b/packages/api/src/user/generations.spec.ts
@@ -0,0 +1,49 @@
+import type { UserGenerationControl } from './generations';
+import { stopUserGenerations } from './generations';
+
+/** Methods read `this`, as the real GenerationJobManager does, so an unbound call throws. */
+class FakeJobManager {
+ private jobs = new Map([
+ ['s1', { createdAt: 7, metadata: { userId: 'u1', terminalPersistencePending: false } }],
+ ['s2', { createdAt: 8, metadata: { userId: 'other' } }],
+ ]);
+
+ readonly aborted: string[] = [];
+
+ async getAccountCleanupJobIdsForUser() {
+ return [...this.jobs.keys()];
+ }
+
+ async getCleanupJob(streamId: string) {
+ return this.jobs.get(streamId);
+ }
+
+ async abortJob(streamId: string) {
+ this.aborted.push(streamId);
+ return { success: true };
+ }
+}
+
+describe('stopUserGenerations', () => {
+ it('stops only the user jobs using a manager whose methods depend on this', async () => {
+ const manager = new FakeJobManager();
+
+ await expect(
+ stopUserGenerations('u1', undefined, manager as unknown as UserGenerationControl),
+ ).resolves.toBeUndefined();
+
+ expect(manager.aborted).toEqual(['s1']);
+ });
+
+ it('throws when a stop is not confirmed', async () => {
+ const manager = new FakeJobManager();
+ manager.abortJob = async function (this: FakeJobManager, streamId: string) {
+ this.aborted.push(streamId);
+ return { success: false };
+ };
+
+ await expect(
+ stopUserGenerations('u1', undefined, manager as unknown as UserGenerationControl),
+ ).rejects.toThrow('could not be confirmed stopped');
+ });
+});
diff --git a/packages/api/src/user/generations.ts b/packages/api/src/user/generations.ts
new file mode 100644
index 0000000000..f57de9a79d
--- /dev/null
+++ b/packages/api/src/user/generations.ts
@@ -0,0 +1,55 @@
+import type { GenerationJob } from '../types/stream';
+import { waitForGenerationPersistence } from '../stream/persistence';
+import { isStopConfirmed } from '../stream/interfaces/IJobStore';
+
+type AbortOutcome = Parameters<typeof isStopConfirmed>[0];
+type CleanupJob = Pick<GenerationJob, 'createdAt' | 'metadata'>;
+
+/** The slice of the generation job manager needed to stop an account's runs. */
+export interface UserGenerationControl {
+ getAccountCleanupJobIdsForUser: (userId: string, tenantId?: string) => Promise<string[]>;
+ getCleanupJob: (streamId: string) => Promise<CleanupJob | undefined>;
+ abortJob: (
+ streamId: string,
+ options: { expectedCreatedAt: number; awaitProviderDrain: boolean },
+ ) => Promise<AbortOutcome>;
+}
+
+/**
+ * Stops the user's in-flight and retained generations and waits for their final persistence,
+ * so nothing is written for the account after its data is erased. The deletion marker must
+ * already be set: a run committed before this listing is stopped here, and one committed after it
+ * checks the marker and retires itself in `createJob`. Throws if a stop cannot be confirmed,
+ * leaving the caller to keep the account and retry.
+ */
+export async function stopUserGenerations(
+ userId: string,
+ tenantId: string | undefined,
+ control: UserGenerationControl,
+): Promise<void> {
+ const streamIds = await control.getAccountCleanupJobIdsForUser(userId, tenantId);
+ const jobs = await Promise.all(
+ streamIds.map(async (streamId) => ({
+ streamId,
+ job: await control.getCleanupJob(streamId),
+ })),
+ );
+ const owned = jobs.filter(
+ ({ job }) =>
+ job?.metadata?.userId === userId &&
+ (!job.metadata.tenantId || job.metadata.tenantId === tenantId),
+ );
+ const results = await Promise.all(
+ owned.map(({ streamId, job }) =>
+ control.abortJob(streamId, { expectedCreatedAt: job!.createdAt, awaitProviderDrain: true }),
+ ),
+ );
+ if (results.some((result) => !isStopConfirmed(result))) {
+ throw new Error('Agent generations could not be confirmed stopped');
+ }
+ await Promise.all(
+ owned.map(({ streamId, job }) =>
+ waitForGenerationPersistence(streamId, job!.createdAt, (id) => control.getCleanupJob(id)),
+ ),
+ );
+}
diff --git a/packages/api/src/user/index.ts b/packages/api/src/user/index.ts
index 1945a8f316..34607738ad 100644
--- a/packages/api/src/user/index.ts
+++ b/packages/api/src/user/index.ts
@@ -1,2 +1,4 @@
export * from './preferences';
export * from './erasure';
+export * from './generations';
+export * from './resolver';
diff --git a/packages/api/src/user/resolver.spec.ts b/packages/api/src/user/resolver.spec.ts
new file mode 100644
index 0000000000..054e7b6082
--- /dev/null
+++ b/packages/api/src/user/resolver.spec.ts
@@ -0,0 +1,210 @@
+import { tenantStorage, runAsSystem } from '@librechat/data-schemas';
+import type { IMongoFile } from '@librechat/data-schemas';
+import type { ErasureResolverDeps } from './resolver';
+import { createErasureResolver } from './resolver';
+
+function createDeps(overrides: Partial<ErasureResolverDeps> = {}) {
+ const disconnectUserConnection = jest.fn().mockResolvedValue(undefined);
+ const deps = {
+ db: { deleteConvos: jest.fn() },
+ getAppConfig: jest
+ .fn()
+ .mockResolvedValue({ endpoints: { agents: { checkpointer: { ttl: 5 } } } }),
+ processDeleteRequest: jest.fn().mockResolvedValue(undefined),
+ invalidateCachedTools: jest.fn().mockResolvedValue(undefined),
+ getMCPManager: jest.fn(() => ({ disconnectUserConnection })),
+ openCheckpointDeletion: jest.fn(),
+ deleteSharedLinks: jest.fn(),
+ withTenant: <T>(tenant: string | undefined, operation: () => Promise<T>) =>
+ tenant ? tenantStorage.run({ tenantId: tenant }, operation) : runAsSystem(operation),
+ ...overrides,
+ } as unknown as ErasureResolverDeps;
+ return { deps, disconnectUserConnection };
+}
+
+const target = { id: 'user-1', role: 'USER', tenantId: 'tenant-1' };
+
+describe('createErasureResolver', () => {
+ it('resolves the target config and its checkpointer', async () => {
+ const { deps } = createDeps();
+ const result = await createErasureResolver(deps)(target);
+
+ expect(deps.getAppConfig).toHaveBeenCalledWith({
+ role: 'USER',
+ userId: 'user-1',
+ tenantId: 'tenant-1',
+ failClosed: true,
+ });
+ expect(result.checkpointer).toEqual({ ttl: 5 });
+ expect(result.deps.openCheckpointDeletion).toBe(deps.openCheckpointDeletion);
+ expect(result.deps.deleteSharedLinks).toBe(deps.deleteSharedLinks);
+ });
+
+ it('passes the loaded external identity so principal resolution does not re-read the user', async () => {
+ const { deps } = createDeps();
+ await createErasureResolver(deps)({ ...target, idOnTheSource: 'ext-1' });
+
+ expect(deps.getAppConfig).toHaveBeenCalledWith(
+ expect.objectContaining({ userId: 'user-1', idOnTheSource: 'ext-1' }),
+ );
+ });
+
+ it('builds the shared-server OAuth hooks for the target account', async () => {
+ const registry = {
+ getServerConfig: jest.fn().mockResolvedValue({ url: 'https://srv' }),
+ resolveAllowlists: jest.fn().mockResolvedValue({}),
+ };
+ const mcpOAuth = {
+ getRegistry: () => registry,
+ getUserPrincipals: jest.fn().mockResolvedValue([]),
+ uninstallOAuthMCP: jest.fn(),
+ };
+ const { deps } = createDeps({ mcpOAuth });
+ const { deps: erasure } = await createErasureResolver(deps)(target);
+
+ await erasure.mcpOAuth?.getServerConfig?.('srv', 'srv-id');
+ await erasure.mcpOAuth?.getUserPrincipals('other-user');
+ await erasure.mcpOAuth?.resolveAllowlists('other-user');
+ await erasure.mcpOAuth?.invalidateUserTools?.('other-user', 'srv');
+ await erasure.mcpOAuth?.disconnectUser?.('other-user', 'srv');
+
+ expect(registry.getServerConfig).toHaveBeenCalledWith('srv', 'user-1');
+ expect(mcpOAuth.getUserPrincipals).toHaveBeenCalledWith({ userId: 'other-user' });
+ expect(registry.resolveAllowlists).toHaveBeenCalledWith({ userId: 'other-user' });
+ expect(deps.invalidateCachedTools).toHaveBeenCalledWith({
+ userId: 'other-user',
+ serverName: 'srv',
+ });
+ await erasure.mcpOAuth?.uninstallOAuthMCP('other-user', 'mcp_srv', {});
+ expect(mcpOAuth.uninstallOAuthMCP).toHaveBeenCalledWith('other-user', 'mcp_srv', {}, undefined);
+ });
+
+ it('runs id-keyed lookups in the target tenant, not the calling admin tenant', async () => {
+ const seen: Record<string, unknown> = {};
+ const record = (key: string) => async () => {
+ seen[key] = tenantStorage.getStore()?.tenantId;
+ return [];
+ };
+ const registry = {
+ getServerConfig: jest.fn(record('config')),
+ resolveAllowlists: jest.fn(record('allowlists')),
+ };
+ const { deps } = createDeps({
+ db: {
+ deleteUserAgents: jest.fn(record('agents')),
+ findMCPServerOAuthTokenUserIds: jest.fn(record('tokens')),
+ findMCPServerViewerEntries: jest.fn(record('viewers')),
+ } as unknown as ErasureResolverDeps['db'],
+ processDeleteRequest: jest.fn(record('files')),
+ invalidateCachedTools: jest.fn(record('fence')),
+ mcpOAuth: {
+ getRegistry: () => registry,
+ getUserPrincipals: jest.fn(record('principals')),
+ uninstallOAuthMCP: jest.fn(record('uninstall')),
+ },
+ } as unknown as Partial<ErasureResolverDeps>);
+ const { deps: erasure } = await createErasureResolver(deps)(target);
+
+ await tenantStorage.run({ tenantId: 'admin-tenant' }, async () => {
+ await erasure.deleteUserAgents('user-1');
+ await erasure.findMCPServerOAuthTokenUserIds?.('srv');
+ await erasure.findMCPServerViewerEntries?.('srv-id');
+ await erasure.deleteStoredFiles([]);
+ await erasure.mcpOAuth?.getServerConfig?.('srv', 'srv-id');
+ await erasure.mcpOAuth?.getUserPrincipals('other');
+ await erasure.mcpOAuth?.resolveAllowlists('other');
+ await erasure.mcpOAuth?.uninstallOAuthMCP('other', 'mcp_srv', {});
+ await erasure.mcpOAuth?.invalidateUserTools?.('other', 'srv');
+ await erasure.mcpOAuth?.disconnectUser?.('other', 'srv');
+ });
+
+ expect(Object.keys(seen)).toHaveLength(9);
+ expect(Object.values(seen).every((tenant) => tenant === 'tenant-1')).toBe(true);
+ });
+
+ it('resolves the target config under the target tenant, not the calling admin tenant', async () => {
+ let seen: string | undefined;
+ const { deps } = createDeps({
+ getAppConfig: jest.fn(async () => {
+ seen = tenantStorage.getStore()?.tenantId;
+ return {};
+ }),
+ });
+
+ await tenantStorage.run({ tenantId: 'admin-tenant' }, () =>
+ createErasureResolver(deps)(target),
+ );
+
+ expect(seen).toBe('tenant-1');
+ });
+
+ it('rejects when the target config cannot be resolved', async () => {
+ const { deps } = createDeps({
+ getAppConfig: jest.fn().mockRejectedValue(new Error('override store down')),
+ });
+ await expect(createErasureResolver(deps)(target)).rejects.toThrow('override store down');
+ });
+
+ it('deletes stored files with a request that carries the target user', async () => {
+ const { deps } = createDeps();
+ const { deps: erasure } = await createErasureResolver(deps)(target);
+ const files = [{ file_id: 'f1' }] as IMongoFile[];
+
+ await erasure.deleteStoredFiles(files);
+
+ expect(deps.processDeleteRequest).toHaveBeenCalledWith({
+ req: expect.objectContaining({
+ user: { id: 'user-1', _id: 'user-1', role: 'USER', tenantId: 'tenant-1' },
+ body: {},
+ query: {},
+ baseUrl: '',
+ }),
+ files,
+ });
+ });
+
+ it('carries the target profile fields for header templates', async () => {
+ const { deps } = createDeps();
+ const { deps: erasure } = await createErasureResolver(deps)({
+ ...target,
+ profile: { email: '[email protected]', name: 'Target' },
+ });
+
+ await erasure.deleteStoredFiles([]);
+
+ expect(deps.processDeleteRequest).toHaveBeenCalledWith({
+ req: expect.objectContaining({
+ user: {
+ email: '[email protected]',
+ name: 'Target',
+ id: 'user-1',
+ _id: 'user-1',
+ role: 'USER',
+ tenantId: 'tenant-1',
+ },
+ }),
+ files: [],
+ });
+ });
+
+ it('scopes MCP runtime cleanup to the target user', async () => {
+ const { deps, disconnectUserConnection } = createDeps();
+ const { deps: erasure } = await createErasureResolver(deps)(target);
+
+ await erasure.invalidateMCPServerTools?.('srv');
+ await erasure.disconnectMCPServer?.('srv');
+
+ expect(deps.invalidateCachedTools).toHaveBeenCalledWith({
+ userId: 'user-1',
+ serverName: 'srv',
+ });
+ expect(disconnectUserConnection).toHaveBeenCalledWith('user-1', 'srv');
+ });
+
+ it('tolerates a missing MCP manager', async () => {
+ const { deps } = createDeps({ getMCPManager: jest.fn(() => undefined) });
+ const { deps: erasure } = await createErasureResolver(deps)(target);
+
+ await expect(erasure.disconnectMCPServer?.('srv')).resolves.toBeUndefined();
+ });
+});
diff --git a/packages/api/src/user/resolver.ts b/packages/api/src/user/resolver.ts
new file mode 100644
index 0000000000..ce32891d10
--- /dev/null
+++ b/packages/api/src/user/resolver.ts
@@ -0,0 +1,141 @@
+import type { TCheckpointerConfig } from 'librechat-data-provider';
+import type { IMongoFile } from '@librechat/data-schemas';
+import type { AdminErasureContext, AdminErasureTarget } from '~/admin/users';
+import type { UserErasureDeps, MCPOAuthErasureHooks } from './erasure';
+
+type ErasureBaseDeps = Omit<
+ UserErasureDeps,
+ | 'openCheckpointDeletion'
+ | 'deleteSharedLinks'
+ | 'deleteStoredFiles'
+ | 'invalidateMCPServerTools'
+ | 'disconnectMCPServer'
+ | 'mcpOAuth'
+>;
+
+/** The stored-file fields deletion reads, without the document members of the persisted model. */
+export type ErasureFile = Pick<
+ IMongoFile,
+ 'file_id' | 'filepath' | 'filename' | 'source' | 'type' | 'bytes' | 'embedded' | 'temp_file_id'
+>;
+
+export interface ErasureResolverDeps {
+ /** Runs an operation in the given tenant's data scope, or system scope when it has none. */
+ withTenant: <T>(tenantId: string | undefined, operation: () => Promise<T>) => Promise<T>;
+ /** Shared-server OAuth cleanup is wired only when the host has an MCP OAuth runtime. */
+ mcpOAuth?: {
+ getRegistry: () => {
+ getServerConfig: (
+ serverName: string,
+ userId: string,
+ ) => ReturnType<NonNullable<MCPOAuthErasureHooks['getServerConfig']>>;
+ resolveAllowlists: (options: {
+ userId: string;
+ }) => ReturnType<MCPOAuthErasureHooks['resolveAllowlists']>;
+ };
+ getUserPrincipals: (params: {
+ userId: string;
+ }) => ReturnType<MCPOAuthErasureHooks['getUserPrincipals']>;
+ uninstallOAuthMCP: MCPOAuthErasureHooks['uninstallOAuthMCP'];
+ };
+ /** Data-layer methods used by the cascade. */
+ db: ErasureBaseDeps;
+ getAppConfig: (options: {
+ role?: string;
+ userId: string;
+ tenantId?: string;
+ idOnTheSource?: string | null;
+ failClosed?: boolean;
+ }) => Promise<{ endpoints?: { agents?: { checkpointer?: TCheckpointerConfig } } } | undefined>;
+ processDeleteRequest: (params: {
+ req: Record<string, unknown>;
+ files: ErasureFile[];
+ }) => Promise<unknown>;
+ invalidateCachedTools: (params: { userId: string; serverName: string }) => Promise<unknown>;
+ getMCPManager: () =>
+ | { disconnectUserConnection: (userId: string, serverName: string) => Promise<unknown> }
+ | null
+ | undefined;
+ openCheckpointDeletion: UserErasureDeps['openCheckpointDeletion'];
+ deleteSharedLinks: UserErasureDeps['deleteSharedLinks'];
+}
+
+/**
+ * Builds the `eraseUserData` dependencies for an account deleted by someone else. Stored files
+ * are removed with a request that carries the target user, since provider clients and
+ * credentials are resolved from the request user, not the calling admin.
+ */
+export function createErasureResolver(deps: ErasureResolverDeps) {
+ return async function resolveErasure({
+ id,
+ role,
+ tenantId,
+ idOnTheSource,
+ profile,
+ }: AdminErasureTarget): Promise<AdminErasureContext> {
+ /** The calling admin's tenant context must not decide whose rows an id-keyed lookup reads. */
+ const inTargetTenant = <T>(operation: () => Promise<T>): Promise<T> =>
+ deps.withTenant(tenantId, operation);
+ /** A failed override lookup must abort the deletion, not fall back to a base config that cannot reach the target's storage. */
+ const appConfig = await inTargetTenant(async () =>
+ deps.getAppConfig({
+ role,
+ userId: id,
+ tenantId,
+ idOnTheSource,
+ failClosed: true,
+ }),
+ );
+ const oauth = deps.mcpOAuth;
+ const fileRequest = {
+ user: { ...profile, id, _id: id, role, tenantId },
+ /** Assistants client setup reads these Express request fields. */
+ baseUrl: '',
+ query: {},
+ body: {},
+ config: appConfig,
+ };
+ return {
+ checkpointer: appConfig?.endpoints?.agents?.checkpointer,
+ deps: {
+ ...deps.db,
+ openCheckpointDeletion: deps.openCheckpointDeletion,
+ deleteSharedLinks: deps.deleteSharedLinks,
+ deleteUserAgents: (userId) => inTargetTenant(async () => deps.db.deleteUserAgents(userId)),
+ findMCPServerOAuthTokenUserIds: (serverName) =>
+ inTargetTenant(async () => deps.db.findMCPServerOAuthTokenUserIds(serverName)),
+ findMCPServerViewerEntries: (serverId) =>
+ inTargetTenant(async () => deps.db.findMCPServerViewerEntries(serverId)),
+ deleteStoredFiles: (files) =>
+ inTargetTenant(async () => deps.processDeleteRequest({ req: fileRequest, files })),
+ invalidateMCPServerTools: (serverName) =>
+ deps.invalidateCachedTools({ userId: id, serverName }),
+ disconnectMCPServer: async (serverName) => {
+ await deps.getMCPManager()?.disconnectUserConnection(id, serverName);
+ },
+ mcpOAuth: oauth && {
+ getServerConfig: (serverName) =>
+ inTargetTenant(async () => oauth.getRegistry().getServerConfig(serverName, id)),
+ getUserPrincipals: (candidateId) =>
+ inTargetTenant(async () => oauth.getUserPrincipals({ userId: candidateId })),
+ resolveAllowlists: (candidateId) =>
+ inTargetTenant(async () =>
+ oauth.getRegistry().resolveAllowlists({ userId: candidateId }),
+ ),
+ uninstallOAuthMCP: (candidateId, pluginKey, cleanupConfig, serverConfig) =>
+ inTargetTenant(async () =>
+ oauth.uninstallOAuthMCP(candidateId, pluginKey, cleanupConfig, serverConfig),
+ ),
+ invalidateUserTools: (candidateId, serverName) =>
+ inTargetTenant(async () =>
+ deps.invalidateCachedTools({ userId: candidateId, serverName }),
+ ),
+ disconnectUser: (candidateId, serverName) =>
+ inTargetTenant(async () =>
+ deps.getMCPManager()?.disconnectUserConnection(candidateId, serverName),
+ ),
+ },
+ },
+ };
+ };
+}