Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🧹 fix: Run the Full Erasure Cascade When an Admin Deletes a User (#17050)
Marco Beretta20d2c0d43d7f
diff --git a/api/server/controllers/UserController.js b/api/server/controllers/UserController.jsindex 3f4bdb66b6..70aa6c2db4 100644--- a/api/server/controllers/UserController.js+++ b/api/server/controllers/UserController.js@@ -15,8 +15,7 @@ const {getWebSearchInstallEntries,getWebSearchUninstallFields,openCheckpointDeletion,- isStopConfirmed,- waitForGenerationPersistence,+ stopUserGenerations,deleteAllSharedLinksWithCleanup,eraseUserData,deleteUserMCPServers,@@ -410,39 +409,7 @@ const deleteUserController = async (req, res) => {if (!(await quiesceUserSchedules(user.id, scheduleSuspensionToken))) {throw new Error('Scheduled executions could not be confirmed stopped');}- const activeAgentRuns = await GenerationJobManager.getAccountCleanupJobIdsForUser(- user.id,- tenantId,- );- const activeAgentJobs = await Promise.all(- activeAgentRuns.map(async (streamId) => ({- streamId,- job: await GenerationJobManager.getCleanupJob(streamId),- })),- );- const ownedAgentJobs = activeAgentJobs.filter(- ({ job }) =>- job?.metadata?.userId === user.id &&- (!job.metadata.tenantId || job.metadata.tenantId === tenantId),- );- const stopResults = await Promise.all(- ownedAgentJobs.map(({ streamId, job }) =>- GenerationJobManager.abortJob(streamId, {- expectedCreatedAt: job.createdAt,- awaitProviderDrain: true,- }),- ),- );- if (stopResults.some((result) => !isStopConfirmed(result))) {- throw new Error('Agent generations could not be confirmed stopped');- }- await Promise.all(- ownedAgentJobs.map(({ streamId, job }) =>- waitForGenerationPersistence(streamId, job.createdAt, (id) =>- GenerationJobManager.getCleanupJob(id),- ),- ),- );+ await stopUserGenerations(user.id, tenantId, GenerationJobManager);const appConfig =req.config ??diff --git a/api/server/controllers/__tests__/deleteUser.spec.js b/api/server/controllers/__tests__/deleteUser.spec.jsindex cbe5dd58ae..13dafb794b 100644--- a/api/server/controllers/__tests__/deleteUser.spec.js+++ b/api/server/controllers/__tests__/deleteUser.spec.js@@ -77,13 +77,9 @@ jest.mock('@librechat/api', () => ({abortJob: (...args) => mockAbortJob(...args),},getOwnedAgentCheckpointScope: jest.fn(() => undefined),- isStopConfirmed: jest.fn(- (result) => result?.success === true || result?.failureReason === 'already_settled',- ),deleteOwnedAgentCheckpoints: jest.fn(),- waitForGenerationPersistence: jest.requireActual(- '../../../../packages/api/src/stream/persistence.ts',- ).waitForGenerationPersistence,+ stopUserGenerations: jest.requireActual('../../../../packages/api/src/user/generations.ts')+ .stopUserGenerations,openCheckpointDeletion: jest.fn(async (userId, tenantId, _root, cfg) => ({remember: jest.fn(),cleanup: () =>diff --git a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.jsindex 1d0790a95a..8b9e2196e1 100644--- a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js+++ b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js@@ -1100,7 +1100,12 @@ describe('ResumableAgentController resume metadata', () => {});it('rejects any session fenced after authentication before generation execution starts', async () => {- mockIsAgentTriggerPrincipalActive.mockResolvedValue(false);+ mockGenerationJobManager.createJob.mockRejectedValueOnce(+ Object.assign(new Error('Account deletion is in progress'), {+ code: 'ACCOUNT_DELETION_IN_PROGRESS',+ status: 409,+ }),+ );const req = {user: { id: 'user-123' },body: {@@ -1118,10 +1123,6 @@ describe('ResumableAgentController resume metadata', () => {await AgentController(req, res, jest.fn(), initializeClient, null);expect(mockGenerationJobManager.createJob).toHaveBeenCalledTimes(1);- expect(mockIsAgentTriggerPrincipalActive).toHaveBeenCalledWith('user-123');- expect(mockGenerationJobManager.createJob.mock.invocationCallOrder[0]).toBeLessThan(- mockIsAgentTriggerPrincipalActive.mock.invocationCallOrder[0],- );expect(mockGenerationJobManager.beginProviderExecution).not.toHaveBeenCalled();expect(res.status).toHaveBeenCalledWith(409);expect(res.json).toHaveBeenCalledWith({@@ -1132,15 +1133,8 @@ describe('ResumableAgentController resume metadata', () => {});expect(initializeClient).not.toHaveBeenCalled();expect(mockAcceptAgentStartupTelemetry).not.toHaveBeenCalled();- expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(- 'conversation-123',- JSON.stringify({- status: 409,- code: 'ACCOUNT_DELETION_IN_PROGRESS',- error: 'Account deletion is in progress',- }),- 1000,- );+ /** createJob retired the refused job itself, so the controller has none to complete. */+ expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();expect(mockGenerationJobManager.releaseGeneration).toHaveBeenCalledWith('user-123','trigger-request',@@ -1239,10 +1233,6 @@ describe('ResumableAgentController resume metadata', () => {await AgentController(req, res, jest.fn(), initializeClient, null);- expect(mockIsAgentTriggerPrincipalActive).toHaveBeenCalledWith('user-123');- expect(mockIsAgentTriggerPrincipalActive.mock.invocationCallOrder[0]).toBeLessThan(- mockGenerationJobManager.beginProviderExecution.mock.invocationCallOrder[0],- );expect(initializeClient).not.toHaveBeenCalled();expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith('conversation-123',diff --git a/api/server/controllers/agents/request.js b/api/server/controllers/agents/request.jsindex 51fa58b533..f6cc1694ee 100644--- a/api/server/controllers/agents/request.js+++ b/api/server/controllers/agents/request.js@@ -100,7 +100,6 @@ const {claimAgentEventActorSuspension,settleAgentEventActorSuspension,stampConvoLastResponse,- isAgentTriggerPrincipalActive,isSubagentOwnerAdmissible,appendConvoMessageReference,} = require('~/models');@@ -1755,18 +1754,8 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTitreq.turnStartedAt = jobCreatedAt;providerExecutionId = job.metadata?.providerExecutionId;- /** Authentication can precede a slow admission path. Recheck the durable- * account-deletion fence after the job is committed but before execution- * starts. This ordering closes both sides of the race for ordinary and- * trigger-scoped sessions: a fence that wins first rejects this run; a- * fence that starts after this read must observe the already-created job- * in account deletion's active-generation drain. */- if (!(await isAgentTriggerPrincipalActive(userId))) {- throw Object.assign(new Error('Account deletion is in progress'), {- code: 'ACCOUNT_DELETION_IN_PROGRESS',- status: 409,- });- }+ /** Account-deletion admission is enforced inside `createJob`, after the job is committed, so a+ * fence that wins first refuses this run and one that starts later finds the job in its drain. */if (req._agentEventBindingParentConversationId != null) {/** The generation job is the durable marker that a deletion on another replica* can abort. Recheck only after that marker exists: either the deletion fencediff --git a/api/server/experimental.js b/api/server/experimental.jsindex c444acd04a..1e168f404d 100644--- a/api/server/experimental.js+++ b/api/server/experimental.js@@ -367,6 +367,7 @@ if (cluster.isMaster) {// but an already-fired scheduled generation can still reach HITL here. Settle// its durable run when the generic approval runtime expires it.GenerationJobManager.setApprovalExpiredHandler(recordExpiredScheduleApproval);+ GenerationJobManager.setAdmissionGuard(agentEventMethods.isAgentTriggerPrincipalActive);GenerationJobManager.setTerminalHostActionHandler(createAgentEventTerminalHandler(agentEventMethods, {resumeDetachedAction: resumeAgentEventDetachedAction,diff --git a/api/server/index.js b/api/server/index.jsindex 469f2863e6..f98c09dfd2 100644--- a/api/server/index.js+++ b/api/server/index.js@@ -139,6 +139,7 @@ const configureGenerationStreams = () => {cleanupOnComplete: !isEnabled(process.env.STREAM_KEEP_COMPLETED_JOBS),});GenerationJobManager.setApprovalExpiredHandler(recordExpiredScheduleApproval);+ GenerationJobManager.setAdmissionGuard(agentEventMethods.isAgentTriggerPrincipalActive);GenerationJobManager.setTerminalHostActionHandler(createAgentEventTerminalHandler(agentEventMethods, {resumeDetachedAction: resumeAgentEventDetachedAction,diff --git a/api/server/routes/admin/users.js b/api/server/routes/admin/users.jsindex 2823abd9f5..efe861248f 100644--- a/api/server/routes/admin/users.js+++ b/api/server/routes/admin/users.js@@ -1,7 +1,14 @@const express = require('express');const mongoose = require('mongoose');-const { createAdminUsersHandlers, revokeUserCodeEnvironmentWorkers } = require('@librechat/api');-const { SystemCapabilities } = require('@librechat/data-schemas');+const {+ GenerationJobManager,+ openCheckpointDeletion,+ deleteAllSharedLinksWithCleanup,+ createErasureResolver,+ createAdminUsersHandlers,+ revokeUserCodeEnvironmentWorkers,+} = require('@librechat/api');+const { SystemCapabilities, runAsSystem, tenantStorage } = require('@librechat/data-schemas');const { requireCapability } = require('~/server/middleware/roles/capabilities');const { requireJwtAuth } = require('~/server/middleware');const {@@ -10,11 +17,37 @@ const {cancelAgentTriggerUserPurge,purgeAgentTriggerDeliveriesForUser,} = require('~/server/services/Agents/triggers');+const { processDeleteRequest } = require('~/server/services/Files/process');+const { invalidateCachedTools } = require('~/server/services/Config/getCachedTools');+const { getMCPManager, getMCPServersRegistry } = require('~/config');+const { maybeUninstallOAuthMCP } = require('~/server/services/MCP/oauthCleanup');+const subagentThreadTaskStore = require('~/server/services/Endpoints/agents/subagentThreadStore');+const {+ quiesceUserSchedules,+ restoreUserSchedulesFromDeletion,+} = require('~/server/services/Schedules');const db = require('~/models');const { getAppConfig, invalidateCodeEnvironmentConfigCache } = require('~/server/services/Config');const router = express.Router();+const resolveErasure = createErasureResolver({+ withTenant: (tenantId, operation) =>+ tenantId ? tenantStorage.run({ tenantId }, operation) : runAsSystem(operation),+ db,+ getAppConfig,+ getMCPManager,+ processDeleteRequest,+ invalidateCachedTools,+ mcpOAuth: {+ getRegistry: getMCPServersRegistry,+ getUserPrincipals: db.getUserPrincipals,+ uninstallOAuthMCP: maybeUninstallOAuthMCP,+ },+ openCheckpointDeletion,+ deleteSharedLinks: (userId) => deleteAllSharedLinksWithCleanup(userId, { awaitCleanup: true }),+});+const requireAdminAccess = requireCapability(SystemCapabilities.ACCESS_ADMIN);const requireReadUsers = requireCapability(SystemCapabilities.READ_USERS);// const requireManageUsers = requireCapability(SystemCapabilities.MANAGE_USERS);@@ -40,7 +73,12 @@ const handlers = createAdminUsersHandlers({deleteConfig: db.deleteConfig,deleteAclEntries: db.deleteAclEntries,deletePasskeysByUser: db.deletePasskeysByUser,- deleteTokens: db.deleteTokens,+ resolveErasure,+ cancelSubagentTasks: (userId, tenantId) =>+ subagentThreadTaskStore.cancelAndDrainForOwner(userId, tenantId),+ quiesceUserSchedules,+ restoreUserSchedulesFromDeletion,+ generationJobs: GenerationJobManager,});router.use(requireJwtAuth, requireAdminAccess);diff --git a/packages/api/src/admin/users.spec.ts b/packages/api/src/admin/users.spec.tsindex 1b969cfef6..3c1aea1303 100644--- a/packages/api/src/admin/users.spec.ts+++ b/packages/api/src/admin/users.spec.ts@@ -5,6 +5,11 @@ import type { Response } from 'express';import type { ServerRequest } from '~/types/http';import type { AdminUsersDeps } from './users';import { createAdminUsersHandlers } from './users';+import { eraseUserData } from '~/user/erasure';++jest.mock('~/user/erasure', () => ({+ eraseUserData: jest.fn().mockResolvedValue(undefined),+}));jest.mock('@librechat/data-schemas', () => ({...jest.requireActual('@librechat/data-schemas'),@@ -68,12 +73,24 @@ function createDeps(overrides: Partial<AdminUsersDeps> = {}): AdminUsersDeps {deleteConfig: jest.fn().mockResolvedValue(null),deleteAclEntries: jest.fn().mockResolvedValue(undefined),deletePasskeysByUser: jest.fn().mockResolvedValue(0),- deleteTokens: jest.fn().mockResolvedValue({ deletedCount: 0 }),+ resolveErasure: jest.fn().mockResolvedValue({ deps: {} }),+ cancelSubagentTasks: jest.fn().mockResolvedValue(undefined),+ quiesceUserSchedules: jest.fn().mockResolvedValue(true),+ restoreUserSchedulesFromDeletion: jest.fn().mockResolvedValue(undefined),+ generationJobs: {+ getAccountCleanupJobIdsForUser: jest.fn().mockResolvedValue([]),+ getCleanupJob: jest.fn().mockResolvedValue(undefined),+ abortJob: jest.fn().mockResolvedValue(null),+ },...overrides,};}describe('createAdminUsersHandlers', () => {+ beforeEach(() => {+ jest.mocked(eraseUserData).mockClear();+ });+describe('listUsers', () => {it('returns paginated users with total count', async () => {const users = [@@ -462,32 +479,118 @@ describe('createAdminUsersHandlers', () => {expect(deps.deletePasskeysByUser).toHaveBeenCalledWith(validUserId);});- it("removes the user's stored credentials before the user", async () => {- const deps = createDeps();+ it('runs the shared erasure cascade for the target before removing the user', async () => {+ const checkpointer = { ttl: 60 };+ const erasureDeps = { marker: true };+ const deps = createDeps({+ findUsers: jest.fn().mockResolvedValue([mockUser({ role: 'USER', tenantId: 'tenant-1' })]),+ resolveErasure: jest.fn().mockResolvedValue({ deps: erasureDeps, checkpointer }),+ });const handlers = createAdminUsersHandlers(deps);const { req, res, status } = createReqRes({ params: { id: validUserId } });await handlers.deleteUser(req, res);expect(status).toHaveBeenCalledWith(200);- expect(deps.deleteTokens).toHaveBeenCalledWith({ userId: validUserId });- expect(jest.mocked(deps.deleteTokens).mock.invocationCallOrder[0]).toBeLessThan(+ expect(deps.resolveErasure).toHaveBeenCalledWith(+ expect.objectContaining({+ id: validUserId,+ role: 'USER',+ tenantId: 'tenant-1',+ idOnTheSource: null,+ profile: expect.objectContaining({ role: 'USER' }),+ }),+ );+ expect(eraseUserData).toHaveBeenCalledWith(+ { userId: validUserId, tenantId: 'tenant-1', checkpointer },+ erasureDeps,+ );+ expect(jest.mocked(eraseUserData).mock.invocationCallOrder[0]).toBeLessThan(jest.mocked(deps.deleteUserById).mock.invocationCallOrder[0],);});- it('keeps the user, and releases the fence, when its credentials cannot be removed', async () => {+ it('keeps the user, and releases the fence, when the erasure cascade fails', async () => {+ jest.mocked(eraseUserData).mockRejectedValueOnce(new Error('storage down'));+ const deps = createDeps();+ const handlers = createAdminUsersHandlers(deps);+ const { req, res, status, json } = createReqRes({ params: { id: validUserId } });++ await handlers.deleteUser(req, res);++ expect(status).toHaveBeenCalledWith(500);+ expect(json).toHaveBeenCalledWith({ error: 'Failed to delete user' });+ expect(deps.deleteUserById).not.toHaveBeenCalled();+ expect(deps.cancelAgentTriggerUserDeletion).toHaveBeenCalled();+ expect(deps.cancelAgentTriggerUserPurge).toHaveBeenCalled();+ });++ it('suspends schedules before erasing and restores them when the deletion fails', async () => {+ jest.mocked(eraseUserData).mockRejectedValueOnce(new Error('storage down'));+ const deps = createDeps();+ const handlers = createAdminUsersHandlers(deps);+ const { req, res, status } = createReqRes({ params: { id: validUserId } });++ await handlers.deleteUser(req, res);++ const token = jest.mocked(deps.quiesceUserSchedules).mock.calls[0][1];+ expect(status).toHaveBeenCalledWith(500);+ expect(deps.restoreUserSchedulesFromDeletion).toHaveBeenCalledWith(validUserId, token);+ expect(+ jest.mocked(deps.restoreUserSchedulesFromDeletion).mock.invocationCallOrder[0],+ ).toBeLessThan(jest.mocked(deps.cancelAgentTriggerUserDeletion).mock.invocationCallOrder[0]);+ });++ it('takes the trigger deletion fence before listing generations to stop', async () => {+ const deps = createDeps();+ const handlers = createAdminUsersHandlers(deps);+ const { req, res } = createReqRes({ params: { id: validUserId } });++ await handlers.deleteUser(req, res);++ expect(+ jest.mocked(deps.beginAgentTriggerUserDeletion).mock.invocationCallOrder[0],+ ).toBeLessThan(+ jest.mocked(deps.generationJobs.getAccountCleanupJobIdsForUser).mock.invocationCallOrder[0],+ );+ });++ it('does not erase when schedules cannot be confirmed stopped', async () => {+ const deps = createDeps({ quiesceUserSchedules: jest.fn().mockResolvedValue(false) });+ const handlers = createAdminUsersHandlers(deps);+ const { req, res, status } = createReqRes({ params: { id: validUserId } });++ await handlers.deleteUser(req, res);++ expect(status).toHaveBeenCalledWith(500);+ expect(eraseUserData).not.toHaveBeenCalled();+ expect(deps.deleteUserById).not.toHaveBeenCalled();+ });++ it('aborts the cleanup jobs of the target and refuses to erase when a stop is unconfirmed', async () => {+ const abortJob = jest.fn().mockResolvedValue(null);const deps = createDeps({- deleteTokens: jest.fn().mockRejectedValue(new Error('mongo down')),+ generationJobs: {+ getAccountCleanupJobIdsForUser: jest.fn().mockResolvedValue(['s1', 's2']),+ getCleanupJob: jest.fn(async (id: string) => ({+ createdAt: 5,+ metadata: { userId: id === 's1' ? validUserId : 'someone-else' },+ })),+ abortJob,+ } as unknown as AdminUsersDeps['generationJobs'],});const handlers = createAdminUsersHandlers(deps);const { req, res, status } = createReqRes({ params: { id: validUserId } });await handlers.deleteUser(req, res);+ expect(abortJob).toHaveBeenCalledTimes(1);+ expect(abortJob).toHaveBeenCalledWith('s1', {+ expectedCreatedAt: 5,+ awaitProviderDrain: true,+ });expect(status).toHaveBeenCalledWith(500);- expect(deps.deleteUserById).not.toHaveBeenCalled();- expect(deps.cancelAgentTriggerUserDeletion).toHaveBeenCalled();+ expect(eraseUserData).not.toHaveBeenCalled();});it('returns success even when cascade cleanup partially fails', async () => {diff --git a/packages/api/src/admin/users.ts b/packages/api/src/admin/users.tsindex e6a1e5f431..ae8ac19c9c 100644--- a/packages/api/src/admin/users.ts+++ b/packages/api/src/admin/users.ts@@ -1,4 +1,5 @@import { Types } from 'mongoose';+import { randomUUID } from 'node:crypto';import { PrincipalType, SystemRoles } from 'librechat-data-provider';import { logger, isValidObjectIdString } from '@librechat/data-schemas';import type {@@ -8,15 +9,61 @@ import type {AdminUserSearchResult,UserDeleteResult,} from '@librechat/data-schemas';+import type { TCheckpointerConfig } from 'librechat-data-provider';import type { FilterQuery } from 'mongoose';import type { Response } from 'express';+import type { UserGenerationControl } from '~/user/generations';+import type { UserErasureDeps } from '~/user/erasure';import type { ServerRequest } from '~/types/http';+import { stopUserGenerations } from '~/user/generations';import { parsePagination } from './pagination';+import { eraseUserData } from '~/user/erasure';const MAX_SEARCH_LENGTH = 200;+const ERASURE_PROFILE_KEYS = [+ 'name',+ 'username',+ 'email',+ 'provider',+ 'googleId',+ 'facebookId',+ 'openidId',+ 'samlId',+ 'ldapId',+ 'githubId',+ 'discordId',+ 'appleId',+ 'emailVerified',+ 'twoFactorEnabled',+ 'termsAccepted',+ 'termsAcceptedAt',+] as const;++const ERASURE_TARGET_FIELDS = ['role', 'tenantId', 'idOnTheSource', ...ERASURE_PROFILE_KEYS].join(+ ' ',+);++/** The target's plain profile fields, which storage header templates such as the user email read. */+export type ErasureTargetProfile = Partial<Pick<IUser, (typeof ERASURE_PROFILE_KEYS)[number]>>;+const USER_LIST_FIELDS = '_id name username email avatar role provider createdAt updatedAt';+/** The target account's identity, from which the caller builds storage and MCP cleanup. */+export interface AdminErasureTarget {+ id: string;+ role?: string;+ tenantId?: string;+ /** External identity key, so principal resolution reuses the loaded user instead of re-reading it. */+ idOnTheSource?: string | null;+ profile?: ErasureTargetProfile;+}++export interface AdminErasureContext {+ deps: UserErasureDeps;+ checkpointer?: TCheckpointerConfig;+}+export interface AdminUsersDeps {findUsers: (searchCriteria: FilterQuery<IUser>,@@ -38,14 +85,22 @@ export interface AdminUsersDeps {cancelAgentTriggerUserPurge: (userId: string, fenceStartedAt: Date) => Promise<boolean>;purgeAgentTriggerDeliveriesForUser: (userId: string) => Promise<void>;revokeUserCodeEnvironmentWorkers?: (userId: string) => Promise<number>;+ /** Cancels and drains the account's subagent thread tasks. */+ cancelSubagentTasks?: (userId: string, tenantId?: string) => Promise<unknown>;/**- * Thin data-layer delete — removes the User document only.- * Full cascade of user-owned resources (conversations, messages, files, tokens, etc.)- * is handled by `UserController.deleteUserController` in the self-delete flow.- * This admin endpoint fences durable triggers around the user commit and currently- * cascades Config and AclEntries.- * A future iteration should consolidate the full cascade into a shared service function.+ * Reversibly suspends the account's schedules under a per-attempt token; false means the+ * executions could not be confirmed stopped. Restored if the deletion fails.*/+ quiesceUserSchedules: (userId: string, token: string) => Promise<boolean>;+ restoreUserSchedulesFromDeletion: (userId: string, token: string) => Promise<unknown>;+ generationJobs: UserGenerationControl;+ /**+ * Resolves the dependencies of the shared `eraseUserData` cascade for the account being+ * deleted, the same set the self-service controller and the delete-user CLI pass. Storage+ * cleanup must run as the target user, not the calling admin.+ */+ resolveErasure: (target: AdminErasureTarget) => Promise<AdminErasureContext>;+ /** Thin data-layer delete: removes the User document after the erasure cascade has run. */deleteUserById: (userId: string) => Promise<UserDeleteResult>;deleteUserCodeEnvironments: (userId: string | Types.ObjectId) => Promise<number>;invalidateCodeEnvironmentConfigCache: (tenantId?: string) => Promise<void>;@@ -62,12 +117,6 @@ export interface AdminUsersDeps {* keep authentication material and can collide with a later registration.*/deletePasskeysByUser: (userId: string) => Promise<number>;- /**- * Stored credentials the user holds, such as a connected account's OAuth grant. Removed before- * the user is, as the self-delete flow does, so a failure leaves the account to retry rather- * than its credentials behind.- */- deleteTokens: (query: { userId: string }) => Promise<unknown>;}export function createAdminUsersHandlers(deps: AdminUsersDeps): {@@ -91,7 +140,11 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {deleteConfig,deleteAclEntries,deletePasskeysByUser,- deleteTokens,+ resolveErasure,+ cancelSubagentTasks,+ quiesceUserSchedules,+ restoreUserSchedulesFromDeletion,+ generationJobs,} = deps;async function listUsersHandler(req: ServerRequest, res: Response) {@@ -173,6 +226,7 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {async function deleteUserHandler(req: ServerRequest, res: Response) {let targetUserId: string | undefined;let triggerDeletionFence: Date | undefined;+ let scheduleSuspensionToken: string | undefined;let userDeleted = false;try {@@ -188,7 +242,7 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {return res.status(403).json({ error: 'Cannot delete your own account' });}- const [targetUser] = await findUsers({ _id: id }, 'role tenantId', { limit: 1 });+ const [targetUser] = await findUsers({ _id: id }, ERASURE_TARGET_FIELDS, { limit: 1 });if (targetUser?.role === SystemRoles.ADMIN) {const adminCount = await countUsers({ role: SystemRoles.ADMIN });if (adminCount <= 1) {@@ -209,7 +263,24 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {await prepareAgentTriggerUserPurge(id, triggerDeletionFence, targetUser?.tenantId);await drainAgentTriggerDeliveriesForUser(id);- await deleteTokens({ userId: id });+ await cancelSubagentTasks?.(id, targetUser?.tenantId);+ scheduleSuspensionToken = randomUUID();+ if (!(await quiesceUserSchedules(id, scheduleSuspensionToken))) {+ throw new Error('Scheduled executions could not be confirmed stopped');+ }+ await stopUserGenerations(id, targetUser?.tenantId, generationJobs);++ const { deps: erasureDeps, checkpointer } = await resolveErasure({+ id,+ role: targetUser?.role,+ tenantId: targetUser?.tenantId,+ idOnTheSource: targetUser?.idOnTheSource ?? null,+ profile: targetUser,+ });+ await eraseUserData(+ { userId: id, tenantId: targetUser?.tenantId, checkpointer },+ erasureDeps,+ );const result = await deleteUserById(id);if (result.deletedCount === 0) {@@ -256,6 +327,17 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {return res.status(200).json({ message: result.message || 'User deleted successfully' });} catch (error) {+ /** Restore before releasing the fence, as the self-service flow does. */+ if (targetUserId != null && scheduleSuspensionToken != null && !userDeleted) {+ try {+ await restoreUserSchedulesFromDeletion(targetUserId, scheduleSuspensionToken);+ } catch (restoreError) {+ logger.error(+ `[adminUsers] failed to restore suspended schedules for user ${targetUserId} under token ${scheduleSuspensionToken}:`,+ restoreError,+ );+ }+ }if (targetUserId != null && triggerDeletionFence != null && !userDeleted) {try {await cancelAgentTriggerUserPurge(targetUserId, triggerDeletionFence);diff --git a/packages/api/src/agents/remote/lifecycle.spec.ts b/packages/api/src/agents/remote/lifecycle.spec.tsindex ea8400d976..cad6969e8e 100644--- a/packages/api/src/agents/remote/lifecycle.spec.ts+++ b/packages/api/src/agents/remote/lifecycle.spec.ts@@ -87,13 +87,10 @@ describe('Agent execution enrollment', () => {expect(beginProviderExecution).not.toHaveBeenCalled();await expect(manager.getCleanupBlockingJobIdsForUser('user-1')).resolves.toEqual([]);- await expect(manager.getJobStore().getJob('chatcmpl-deleting')).resolves.toMatchObject({- status: 'error',- providerDrained: true,- });+ await expect(manager.getJobStore().getJob('chatcmpl-deleting')).resolves.toBeNull();});- it('preserves principal-check infrastructure failures after retiring the run', async () => {+ it('reports principal-check infrastructure failures with a sanitized error after retiring the run', async () => {const infrastructureError = new Error('principal store unavailable');await expect(@@ -104,7 +101,7 @@ describe('Agent execution enrollment', () => {},{ manager },),- ).rejects.toBe(infrastructureError);+ ).rejects.toMatchObject({ code: 'GENERATION_ADMISSION_UNAVAILABLE' });await expect(manager.getCleanupBlockingJobIdsForUser('user-1')).resolves.toEqual([]);});diff --git a/packages/api/src/agents/remote/lifecycle.ts b/packages/api/src/agents/remote/lifecycle.tsindex b80aabbdb4..3191c75335 100644--- a/packages/api/src/agents/remote/lifecycle.ts+++ b/packages/api/src/agents/remote/lifecycle.ts@@ -1,6 +1,6 @@import type { GenerationJobManagerClass } from '~/stream';import type { GenerationJob } from '~/types/stream';-import { GenerationJobManager } from '~/stream';+import { GenerationJobManager, GenerationAdmissionDeniedError } from '~/stream';const ACCOUNT_DELETION_ERROR = 'Account deletion is in progress';const REMOTE_EXECUTION_ERROR = 'Remote agent execution failed';@@ -190,38 +190,33 @@ export class AgentExecutionEnrollment {}}-async function retireRejectedEnrollment(- manager: GenerationJobManagerClass,- job: GenerationJob,-): Promise<void> {- await manager.completeJob(job.streamId, ACCOUNT_DELETION_ERROR, job.createdAt);-}-export async function enrollAgentExecution(params: AgentExecutionEnrollmentParams,deps: AgentExecutionEnrollmentDeps = { manager: GenerationJobManager },): Promise<AgentExecutionEnrollment> {const { runId, userId, conversationId, agentId, protocol, isPrincipalActive } = params;- const job = await deps.manager.createJob(runId, userId, conversationId, {- initialMetadata: {- agent_id: agentId,- endpoint: protocol,- model: agentId,- responseMessageId: runId,- },- });-- let active = false;+ let job: GenerationJob;try {- active = await isPrincipalActive(userId);+ /** The principal predicate replaces the default guard, so admission is one post-commit read+ * and a refused run takes the manager's retire-and-purge path. */+ job = await deps.manager.createJob(runId, userId, conversationId, {+ admissionCheck: isPrincipalActive,+ initialMetadata: {+ agent_id: agentId,+ endpoint: protocol,+ model: agentId,+ responseMessageId: runId,+ },+ });} catch (error) {- await retireRejectedEnrollment(deps.manager, job).catch(() => undefined);+ if (error instanceof GenerationAdmissionDeniedError) {+ throw new AgentExecutionAdmissionError(+ ACCOUNT_DELETION_ERROR,+ 'ACCOUNT_DELETION_IN_PROGRESS',+ );+ }throw error;}- if (!active) {- await retireRejectedEnrollment(deps.manager, job);- throw new AgentExecutionAdmissionError(ACCOUNT_DELETION_ERROR, 'ACCOUNT_DELETION_IN_PROGRESS');- }return new AgentExecutionEnrollment(deps.manager, job);}diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.tsindex 9f721c2a3d..48353dbdce 100644--- a/packages/api/src/stream/GenerationJobManager.ts+++ b/packages/api/src/stream/GenerationJobManager.ts@@ -44,28 +44,31 @@ import type { RecoveredSteerPayload } from './SteerRecovery';import type { SteerContentView } from './SteeringLifecycle';import type { GenerationJobStore } from '~/app/metrics';import type * as t from '~/types';-import {- recordGenerationStreamEarlyBufferOverflow,- recordGenerationStreamResumePendingEvents,- recordGenerationStreamSubscription,- recordGenerationStreamAttachment,- recordGenerationStreamRecovery,- setGenerationJobsInFlight,- recordGenerationJob,-} from '~/app/metrics';import {GenerationPublicationFencedError,JobCreationSupersededError,JobPredecessorMismatchError,+ GenerationAdmissionDeniedError,+ GenerationAdmissionUnavailableError,isPendingActionStale,isPendingActionExpired,PAUSE_PERSISTENCE_TIMEOUT_ERROR,PROVIDER_DRAIN_TIMEOUT_MS,STEER_QUEUE_MAX_DEPTH,} from './interfaces/IJobStore';+import {+ recordGenerationStreamEarlyBufferOverflow,+ recordGenerationStreamResumePendingEvents,+ recordGenerationStreamSubscription,+ recordGenerationStreamAttachment,+ recordGenerationStreamRecovery,+ setGenerationJobsInFlight,+ recordGenerationJob,+} from '~/app/metrics';import { isRecoveredSteerPayload, RecoveredSteerPayloadMismatchError } from './SteerRecovery';import { projectTerminalEvent } from './terminalProjection';import { assertJobStoreV2 } from './jobStoreCapabilities';+import { getSafeErrorMetadata } from '~/utils/errors';/*** Tombstone budget per generation. Twice the queue depth so a full queue's@@ -143,6 +146,7 @@ const IDEMPOTENCY_TTL_SECONDS = 25 * 60 * 60;const LEGACY_IDEMPOTENCY_TTL_SECONDS = 26 * 60 * 60;const OAUTH_TOOL_CALL_PREFIX = `oauth${Constants.mcp_delimiter}`;const SHUTDOWN_SUBSCRIBER_ERROR = 'Server is shutting down';+const ACCOUNT_DELETION_JOB_ERROR = 'Generation stopped because the account is being deleted';const SHUTDOWN_JOB_ERROR = 'Generation interrupted because its server shut down';const SHUTTING_DOWN_ERROR = 'Generation job manager is shutting down';/** Internal transport signal: durable terminal state exists, but its DONE@@ -662,6 +666,8 @@ export type TerminalHostActionHandler = () => void | Promise<void>;export interface CreateGenerationJobOptions {+ /** Replaces the installed admission guard for this job, for callers with a stricter predicate. */+ admissionCheck?: (userId: string) => Promise<boolean>;startupTelemetry?: AgentStartupTelemetry;initialMetadata?: Partial<t.GenerationJobMetadata>;/** A terminal steer being handed off as this new normal turn. The store@@ -886,6 +892,8 @@ type DeferredDelivery =* ```*/class GenerationJobManagerClass {+ private admissionGuard?: (userId: string) => Promise<boolean>;+/** Job metadata + content state storage - swappable for Redis, etc. */private jobStore: IJobStoreV2;/** Guarded human-review lifecycle (pause / resolve / expire) over the store. */@@ -1142,6 +1150,38 @@ class GenerationJobManagerClass {this.approvalExpiredHandler = handler;}+ /**+ * Installs the account admission check, which `createJob` runs before it creates any durable+ * state. The caller supplies it, keeping the stream package storage-free.+ */+ setAdmissionGuard(guard?: (userId: string) => Promise<boolean>): void {+ this.admissionGuard = guard;+ }++ private async assertAdmitted(+ streamId: string,+ userId: string,+ options: CreateGenerationJobOptions,+ ): Promise<void> {+ const admissionCheck = options.admissionCheck ?? this.admissionGuard;+ if (admissionCheck == null) {+ return;+ }+ let admitted: boolean;+ try {+ admitted = await admissionCheck(userId);+ } catch (error) {+ logger.error(+ `[GenerationJobManager] Admission check failed for ${streamId}:`,+ getSafeErrorMetadata(error),+ );+ throw new GenerationAdmissionUnavailableError();+ }+ if (!admitted) {+ throw new GenerationAdmissionDeniedError();+ }+ }+/** Installs a durable, generation-fenced terminal lifecycle adapter. */setTerminalHostActionHandler(handler?: TerminalHostActionHandler): void {this.terminalHostActionHandler = handler;@@ -2540,6 +2580,9 @@ class GenerationJobManagerClass {throw new Error('Invalid active generation predecessor policy');}+ /** Refuse before any durable state exists; the common case leaves nothing to clean up. */+ await this.assertAdmitted(streamId, userId, options);+const tenantId = getTenantId();const safeTenantId = tenantId && tenantId !== SYSTEM_TENANT_ID ? tenantId : undefined;const creationAttemptId = randomUUID();@@ -2731,7 +2774,21 @@ class GenerationJobManagerClass {}throw new Error('Generation job was replaced during initialization');}- if (this.shuttingDown) {+ /**+ * Race-free admission: account deletion sets its marker first and then lists this user's jobs.+ * A job committed before that listing is stopped by the drain; a job committed after it+ * checks the marker here, after the marker was set, and retires itself through the normal+ * terminal path. The marker is therefore always seen by one side or the other.+ */+ let postCommitRefusal: Error | undefined;+ if (!this.shuttingDown) {+ try {+ await this.assertAdmitted(streamId, userId, options);+ } catch (error) {+ postCommitRefusal = error as Error;+ }+ }+ if (this.shuttingDown || postCommitRefusal != null) {await this.processDurableReplacementReceipts(streamId, jobData, conversationId).catch((handoffError) => {logger.error(@@ -2741,20 +2798,35 @@ class GenerationJobManagerClass {return false;},);- await this.completeJob(streamId, SHUTDOWN_JOB_ERROR, jobData.createdAt);- if (jobData.providerExecutionId) {- await this.markProviderExecutionDrained(+ try {+ await this.completeJob(streamId,+ postCommitRefusal != null ? ACCOUNT_DELETION_JOB_ERROR : SHUTDOWN_JOB_ERROR,jobData.createdAt,- jobData.providerExecutionId,);+ if (jobData.providerExecutionId) {+ await this.markProviderExecutionDrained(+ streamId,+ jobData.createdAt,+ jobData.providerExecutionId,+ );+ }+ const replacedLocal = this.runtimeState.get(streamId);+ if (replacedLocal != null && replacedLocal.createdAt < jobData.createdAt) {+ replacedLocal.abortController.abort();+ this.retireStoppedPredecessorRuntime(streamId, jobData.createdAt);+ }+ } catch (retireError) {+ if (postCommitRefusal == null) {+ throw retireError;+ }+ logger.error(+ `[GenerationJobManager] Failed to retire refused generation ${streamId}:`,+ getSafeErrorMetadata(retireError),+ );+ throw new GenerationAdmissionUnavailableError();}- const replacedLocal = this.runtimeState.get(streamId);- if (replacedLocal != null && replacedLocal.createdAt < jobData.createdAt) {- replacedLocal.abortController.abort();- this.retireStoppedPredecessorRuntime(streamId, jobData.createdAt);- }- throw new Error(SHUTTING_DOWN_ERROR);+ throw postCommitRefusal ?? new Error(SHUTTING_DOWN_ERROR);}if (!(await this.processDurableReplacementReceipts(streamId, jobData, conversationId))) {diff --git a/packages/api/src/stream/__tests__/admissionGuard.spec.ts b/packages/api/src/stream/__tests__/admissionGuard.spec.tsnew file mode 100644index 0000000000..b3a78b85b6--- /dev/null+++ b/packages/api/src/stream/__tests__/admissionGuard.spec.ts@@ -0,0 +1,112 @@+import {+ GenerationAdmissionDeniedError,+ GenerationAdmissionUnavailableError,+} from '~/stream/interfaces/IJobStore';+import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport';+import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore';+import { GenerationJobManagerClass } from '~/stream/GenerationJobManager';+import { stopUserGenerations } from '~/user/generations';++describe('GenerationJobManager admission guard', () => {+ let manager: GenerationJobManagerClass;+ let store: InMemoryJobStore;++ beforeEach(() => {+ store = new InMemoryJobStore({ ttlAfterComplete: 60000 });+ manager = new GenerationJobManagerClass();+ manager.configure({ jobStore: store, eventTransport: new InMemoryEventTransport() });+ manager.initialize();+ });++ afterEach(async () => {+ await manager.destroy();+ });++ it('refuses before creating any job state while the account deletion has begun', async () => {+ manager.setAdmissionGuard(async () => false);++ await expect(manager.createJob('denied', 'user-1', 'denied')).rejects.toBeInstanceOf(+ GenerationAdmissionDeniedError,+ );+ await expect(manager.createJob('denied', 'user-1', 'denied')).rejects.toMatchObject({+ code: 'ACCOUNT_DELETION_IN_PROGRESS',+ status: 409,+ message: 'Account deletion is in progress',+ });+ expect(await store.getJob('denied')).toBeNull();+ });++ it('refuses with a sanitized code when the guard fails and creates nothing', async () => {+ manager.setAdmissionGuard(async () => {+ throw new Error('mongodb://internal-host:27017 lookup down');+ });++ const error = await manager.createJob('failed', 'user-1', 'failed').catch((e) => e);++ expect(error).toBeInstanceOf(GenerationAdmissionUnavailableError);+ expect(error).toMatchObject({+ code: 'GENERATION_ADMISSION_UNAVAILABLE',+ message: 'Generation admission could not be verified',+ });+ expect(await store.getJob('failed')).toBeNull();+ });++ it('uses a per-job admission check in place of the installed guard', async () => {+ const guard = jest.fn(async () => true);+ manager.setAdmissionGuard(guard);++ await expect(+ manager.createJob('own-check', 'user-1', 'own-check', { admissionCheck: async () => false }),+ ).rejects.toBeInstanceOf(GenerationAdmissionDeniedError);+ expect(guard).not.toHaveBeenCalled();+ });++ it('admits jobs while the account is active', async () => {+ manager.setAdmissionGuard(async () => true);++ await expect(manager.createJob('ok', 'user-1', 'ok')).resolves.toMatchObject({+ streamId: 'ok',+ });+ });++ it('retires a job committed after the marker was set and fails with the stable code', async () => {+ let marker = false;+ manager.setAdmissionGuard(async () => !marker);+ const created = jest.spyOn(store, 'createJob');+ created.mockImplementationOnce(async (...args) => {+ marker = true;+ return InMemoryJobStore.prototype.createJob.apply(store, args);+ });++ await expect(manager.createJob('late', 'user-1', 'late')).rejects.toBeInstanceOf(+ GenerationAdmissionDeniedError,+ );+ expect((await store.getJob('late'))?.status).toBe('error');+ });++ it('maps a retirement failure after a refused commit to the sanitized error', async () => {+ let marker = false;+ manager.setAdmissionGuard(async () => !marker);+ jest.spyOn(store, 'createJob').mockImplementationOnce(async (...args) => {+ marker = true;+ return InMemoryJobStore.prototype.createJob.apply(store, args);+ });+ jest.spyOn(manager, 'completeJob').mockRejectedValueOnce(new Error('redis://internal:6379'));++ await expect(manager.createJob('retire-fail', 'user-1', 'retire-fail')).rejects.toMatchObject({+ code: 'GENERATION_ADMISSION_UNAVAILABLE',+ message: 'Generation admission could not be verified',+ });+ });++ it('stops a job committed before the deletion listing', async () => {+ let marker = false;+ manager.setAdmissionGuard(async () => !marker);+ await manager.createJob('before', 'user-1', 'before');++ marker = true;+ await stopUserGenerations('user-1', undefined, manager);++ expect((await store.getJob('before'))?.status).not.toBe('running');+ });+});diff --git a/packages/api/src/stream/index.ts b/packages/api/src/stream/index.tsindex c26c310b7c..b54ffdcb0d 100644--- a/packages/api/src/stream/index.ts+++ b/packages/api/src/stream/index.ts@@ -31,6 +31,8 @@ export type {export {GenerationPublicationFencedError,JobPredecessorMismatchError,+ GenerationAdmissionDeniedError,+ GenerationAdmissionUnavailableError,isPendingActionExpired,isPendingActionStale,} from './interfaces/IJobStore';diff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.tsindex d160933229..f9e040252c 100644--- a/packages/api/src/stream/interfaces/IJobStore.ts+++ b/packages/api/src/stream/interfaces/IJobStore.ts@@ -460,6 +460,28 @@ export class JobPredecessorMismatchError extends Error {}}+/** Thrown by `createJob` when the owner's account deletion began before the job could be admitted. */+export class GenerationAdmissionDeniedError extends Error {+ readonly code = 'ACCOUNT_DELETION_IN_PROGRESS';+ readonly status = 409;++ constructor() {+ super('Account deletion is in progress');+ this.name = 'GenerationAdmissionDeniedError';+ }+}++/** Thrown by `createJob` when the account admission check itself failed; carries no lookup diagnostics. */+export class GenerationAdmissionUnavailableError extends Error {+ readonly code = 'GENERATION_ADMISSION_UNAVAILABLE';+ readonly status = 503;++ constructor() {+ super('Generation admission could not be verified');+ this.name = 'GenerationAdmissionUnavailableError';+ }+}+export type JobMetadataPatch = Partial<Pick<SerializableJobData,diff --git a/packages/api/src/user/generations.spec.ts b/packages/api/src/user/generations.spec.tsnew file mode 100644index 0000000000..da2681a3fe--- /dev/null+++ b/packages/api/src/user/generations.spec.ts@@ -0,0 +1,49 @@+import type { UserGenerationControl } from './generations';+import { stopUserGenerations } from './generations';++/** Methods read `this`, as the real GenerationJobManager does, so an unbound call throws. */+class FakeJobManager {+ private jobs = new Map([+ ['s1', { createdAt: 7, metadata: { userId: 'u1', terminalPersistencePending: false } }],+ ['s2', { createdAt: 8, metadata: { userId: 'other' } }],+ ]);++ readonly aborted: string[] = [];++ async getAccountCleanupJobIdsForUser() {+ return [...this.jobs.keys()];+ }++ async getCleanupJob(streamId: string) {+ return this.jobs.get(streamId);+ }++ async abortJob(streamId: string) {+ this.aborted.push(streamId);+ return { success: true };+ }+}++describe('stopUserGenerations', () => {+ it('stops only the user jobs using a manager whose methods depend on this', async () => {+ const manager = new FakeJobManager();++ await expect(+ stopUserGenerations('u1', undefined, manager as unknown as UserGenerationControl),+ ).resolves.toBeUndefined();++ expect(manager.aborted).toEqual(['s1']);+ });++ it('throws when a stop is not confirmed', async () => {+ const manager = new FakeJobManager();+ manager.abortJob = async function (this: FakeJobManager, streamId: string) {+ this.aborted.push(streamId);+ return { success: false };+ };++ await expect(+ stopUserGenerations('u1', undefined, manager as unknown as UserGenerationControl),+ ).rejects.toThrow('could not be confirmed stopped');+ });+});diff --git a/packages/api/src/user/generations.ts b/packages/api/src/user/generations.tsnew file mode 100644index 0000000000..f57de9a79d--- /dev/null+++ b/packages/api/src/user/generations.ts@@ -0,0 +1,55 @@+import type { GenerationJob } from '../types/stream';+import { waitForGenerationPersistence } from '../stream/persistence';+import { isStopConfirmed } from '../stream/interfaces/IJobStore';++type AbortOutcome = Parameters<typeof isStopConfirmed>[0];+type CleanupJob = Pick<GenerationJob, 'createdAt' | 'metadata'>;++/** The slice of the generation job manager needed to stop an account's runs. */+export interface UserGenerationControl {+ getAccountCleanupJobIdsForUser: (userId: string, tenantId?: string) => Promise<string[]>;+ getCleanupJob: (streamId: string) => Promise<CleanupJob | undefined>;+ abortJob: (+ streamId: string,+ options: { expectedCreatedAt: number; awaitProviderDrain: boolean },+ ) => Promise<AbortOutcome>;+}++/**+ * Stops the user's in-flight and retained generations and waits for their final persistence,+ * so nothing is written for the account after its data is erased. The deletion marker must+ * already be set: a run committed before this listing is stopped here, and one committed after it+ * checks the marker and retires itself in `createJob`. Throws if a stop cannot be confirmed,+ * leaving the caller to keep the account and retry.+ */+export async function stopUserGenerations(+ userId: string,+ tenantId: string | undefined,+ control: UserGenerationControl,+): Promise<void> {+ const streamIds = await control.getAccountCleanupJobIdsForUser(userId, tenantId);+ const jobs = await Promise.all(+ streamIds.map(async (streamId) => ({+ streamId,+ job: await control.getCleanupJob(streamId),+ })),+ );+ const owned = jobs.filter(+ ({ job }) =>+ job?.metadata?.userId === userId &&+ (!job.metadata.tenantId || job.metadata.tenantId === tenantId),+ );+ const results = await Promise.all(+ owned.map(({ streamId, job }) =>+ control.abortJob(streamId, { expectedCreatedAt: job!.createdAt, awaitProviderDrain: true }),+ ),+ );+ if (results.some((result) => !isStopConfirmed(result))) {+ throw new Error('Agent generations could not be confirmed stopped');+ }+ await Promise.all(+ owned.map(({ streamId, job }) =>+ waitForGenerationPersistence(streamId, job!.createdAt, (id) => control.getCleanupJob(id)),+ ),+ );+}diff --git a/packages/api/src/user/index.ts b/packages/api/src/user/index.tsindex 1945a8f316..34607738ad 100644--- a/packages/api/src/user/index.ts+++ b/packages/api/src/user/index.ts@@ -1,2 +1,4 @@export * from './preferences';export * from './erasure';+export * from './generations';+export * from './resolver';diff --git a/packages/api/src/user/resolver.spec.ts b/packages/api/src/user/resolver.spec.tsnew file mode 100644index 0000000000..054e7b6082--- /dev/null+++ b/packages/api/src/user/resolver.spec.ts@@ -0,0 +1,210 @@+import { tenantStorage, runAsSystem } from '@librechat/data-schemas';+import type { IMongoFile } from '@librechat/data-schemas';+import type { ErasureResolverDeps } from './resolver';+import { createErasureResolver } from './resolver';++function createDeps(overrides: Partial<ErasureResolverDeps> = {}) {+ const disconnectUserConnection = jest.fn().mockResolvedValue(undefined);+ const deps = {+ db: { deleteConvos: jest.fn() },+ getAppConfig: jest+ .fn()+ .mockResolvedValue({ endpoints: { agents: { checkpointer: { ttl: 5 } } } }),+ processDeleteRequest: jest.fn().mockResolvedValue(undefined),+ invalidateCachedTools: jest.fn().mockResolvedValue(undefined),+ getMCPManager: jest.fn(() => ({ disconnectUserConnection })),+ openCheckpointDeletion: jest.fn(),+ deleteSharedLinks: jest.fn(),+ withTenant: <T>(tenant: string | undefined, operation: () => Promise<T>) =>+ tenant ? tenantStorage.run({ tenantId: tenant }, operation) : runAsSystem(operation),+ ...overrides,+ } as unknown as ErasureResolverDeps;+ return { deps, disconnectUserConnection };+}++const target = { id: 'user-1', role: 'USER', tenantId: 'tenant-1' };++describe('createErasureResolver', () => {+ it('resolves the target config and its checkpointer', async () => {+ const { deps } = createDeps();+ const result = await createErasureResolver(deps)(target);++ expect(deps.getAppConfig).toHaveBeenCalledWith({+ role: 'USER',+ userId: 'user-1',+ tenantId: 'tenant-1',+ failClosed: true,+ });+ expect(result.checkpointer).toEqual({ ttl: 5 });+ expect(result.deps.openCheckpointDeletion).toBe(deps.openCheckpointDeletion);+ expect(result.deps.deleteSharedLinks).toBe(deps.deleteSharedLinks);+ });++ it('passes the loaded external identity so principal resolution does not re-read the user', async () => {+ const { deps } = createDeps();+ await createErasureResolver(deps)({ ...target, idOnTheSource: 'ext-1' });++ expect(deps.getAppConfig).toHaveBeenCalledWith(+ expect.objectContaining({ userId: 'user-1', idOnTheSource: 'ext-1' }),+ );+ });++ it('builds the shared-server OAuth hooks for the target account', async () => {+ const registry = {+ getServerConfig: jest.fn().mockResolvedValue({ url: 'https://srv' }),+ resolveAllowlists: jest.fn().mockResolvedValue({}),+ };+ const mcpOAuth = {+ getRegistry: () => registry,+ getUserPrincipals: jest.fn().mockResolvedValue([]),+ uninstallOAuthMCP: jest.fn(),+ };+ const { deps } = createDeps({ mcpOAuth });+ const { deps: erasure } = await createErasureResolver(deps)(target);++ await erasure.mcpOAuth?.getServerConfig?.('srv', 'srv-id');+ await erasure.mcpOAuth?.getUserPrincipals('other-user');+ await erasure.mcpOAuth?.resolveAllowlists('other-user');+ await erasure.mcpOAuth?.invalidateUserTools?.('other-user', 'srv');+ await erasure.mcpOAuth?.disconnectUser?.('other-user', 'srv');++ expect(registry.getServerConfig).toHaveBeenCalledWith('srv', 'user-1');+ expect(mcpOAuth.getUserPrincipals).toHaveBeenCalledWith({ userId: 'other-user' });+ expect(registry.resolveAllowlists).toHaveBeenCalledWith({ userId: 'other-user' });+ expect(deps.invalidateCachedTools).toHaveBeenCalledWith({+ userId: 'other-user',+ serverName: 'srv',+ });+ await erasure.mcpOAuth?.uninstallOAuthMCP('other-user', 'mcp_srv', {});+ expect(mcpOAuth.uninstallOAuthMCP).toHaveBeenCalledWith('other-user', 'mcp_srv', {}, undefined);+ });++ it('runs id-keyed lookups in the target tenant, not the calling admin tenant', async () => {+ const seen: Record<string, unknown> = {};+ const record = (key: string) => async () => {+ seen[key] = tenantStorage.getStore()?.tenantId;+ return [];+ };+ const registry = {+ getServerConfig: jest.fn(record('config')),+ resolveAllowlists: jest.fn(record('allowlists')),+ };+ const { deps } = createDeps({+ db: {+ deleteUserAgents: jest.fn(record('agents')),+ findMCPServerOAuthTokenUserIds: jest.fn(record('tokens')),+ findMCPServerViewerEntries: jest.fn(record('viewers')),+ } as unknown as ErasureResolverDeps['db'],+ processDeleteRequest: jest.fn(record('files')),+ invalidateCachedTools: jest.fn(record('fence')),+ mcpOAuth: {+ getRegistry: () => registry,+ getUserPrincipals: jest.fn(record('principals')),+ uninstallOAuthMCP: jest.fn(record('uninstall')),+ },+ } as unknown as Partial<ErasureResolverDeps>);+ const { deps: erasure } = await createErasureResolver(deps)(target);++ await tenantStorage.run({ tenantId: 'admin-tenant' }, async () => {+ await erasure.deleteUserAgents('user-1');+ await erasure.findMCPServerOAuthTokenUserIds?.('srv');+ await erasure.findMCPServerViewerEntries?.('srv-id');+ await erasure.deleteStoredFiles([]);+ await erasure.mcpOAuth?.getServerConfig?.('srv', 'srv-id');+ await erasure.mcpOAuth?.getUserPrincipals('other');+ await erasure.mcpOAuth?.resolveAllowlists('other');+ await erasure.mcpOAuth?.uninstallOAuthMCP('other', 'mcp_srv', {});+ await erasure.mcpOAuth?.invalidateUserTools?.('other', 'srv');+ await erasure.mcpOAuth?.disconnectUser?.('other', 'srv');+ });++ expect(Object.keys(seen)).toHaveLength(9);+ expect(Object.values(seen).every((tenant) => tenant === 'tenant-1')).toBe(true);+ });++ it('resolves the target config under the target tenant, not the calling admin tenant', async () => {+ let seen: string | undefined;+ const { deps } = createDeps({+ getAppConfig: jest.fn(async () => {+ seen = tenantStorage.getStore()?.tenantId;+ return {};+ }),+ });++ await tenantStorage.run({ tenantId: 'admin-tenant' }, () =>+ createErasureResolver(deps)(target),+ );++ expect(seen).toBe('tenant-1');+ });++ it('rejects when the target config cannot be resolved', async () => {+ const { deps } = createDeps({+ getAppConfig: jest.fn().mockRejectedValue(new Error('override store down')),+ });+ await expect(createErasureResolver(deps)(target)).rejects.toThrow('override store down');+ });++ it('deletes stored files with a request that carries the target user', async () => {+ const { deps } = createDeps();+ const { deps: erasure } = await createErasureResolver(deps)(target);+ const files = [{ file_id: 'f1' }] as IMongoFile[];++ await erasure.deleteStoredFiles(files);++ expect(deps.processDeleteRequest).toHaveBeenCalledWith({+ req: expect.objectContaining({+ user: { id: 'user-1', _id: 'user-1', role: 'USER', tenantId: 'tenant-1' },+ body: {},+ query: {},+ baseUrl: '',+ }),+ files,+ });+ });++ it('carries the target profile fields for header templates', async () => {+ const { deps } = createDeps();+ const { deps: erasure } = await createErasureResolver(deps)({+ ...target,+ profile: { email: '[email protected]', name: 'Target' },+ });++ await erasure.deleteStoredFiles([]);++ expect(deps.processDeleteRequest).toHaveBeenCalledWith({+ req: expect.objectContaining({+ user: {+ email: '[email protected]',+ name: 'Target',+ id: 'user-1',+ _id: 'user-1',+ role: 'USER',+ tenantId: 'tenant-1',+ },+ }),+ files: [],+ });+ });++ it('scopes MCP runtime cleanup to the target user', async () => {+ const { deps, disconnectUserConnection } = createDeps();+ const { deps: erasure } = await createErasureResolver(deps)(target);++ await erasure.invalidateMCPServerTools?.('srv');+ await erasure.disconnectMCPServer?.('srv');++ expect(deps.invalidateCachedTools).toHaveBeenCalledWith({+ userId: 'user-1',+ serverName: 'srv',+ });+ expect(disconnectUserConnection).toHaveBeenCalledWith('user-1', 'srv');+ });++ it('tolerates a missing MCP manager', async () => {+ const { deps } = createDeps({ getMCPManager: jest.fn(() => undefined) });+ const { deps: erasure } = await createErasureResolver(deps)(target);++ await expect(erasure.disconnectMCPServer?.('srv')).resolves.toBeUndefined();+ });+});diff --git a/packages/api/src/user/resolver.ts b/packages/api/src/user/resolver.tsnew file mode 100644index 0000000000..ce32891d10--- /dev/null+++ b/packages/api/src/user/resolver.ts@@ -0,0 +1,141 @@+import type { TCheckpointerConfig } from 'librechat-data-provider';+import type { IMongoFile } from '@librechat/data-schemas';+import type { AdminErasureContext, AdminErasureTarget } from '~/admin/users';+import type { UserErasureDeps, MCPOAuthErasureHooks } from './erasure';++type ErasureBaseDeps = Omit<+ UserErasureDeps,+ | 'openCheckpointDeletion'+ | 'deleteSharedLinks'+ | 'deleteStoredFiles'+ | 'invalidateMCPServerTools'+ | 'disconnectMCPServer'+ | 'mcpOAuth'+>;++/** The stored-file fields deletion reads, without the document members of the persisted model. */+export type ErasureFile = Pick<+ IMongoFile,+ 'file_id' | 'filepath' | 'filename' | 'source' | 'type' | 'bytes' | 'embedded' | 'temp_file_id'+>;++export interface ErasureResolverDeps {+ /** Runs an operation in the given tenant's data scope, or system scope when it has none. */+ withTenant: <T>(tenantId: string | undefined, operation: () => Promise<T>) => Promise<T>;+ /** Shared-server OAuth cleanup is wired only when the host has an MCP OAuth runtime. */+ mcpOAuth?: {+ getRegistry: () => {+ getServerConfig: (+ serverName: string,+ userId: string,+ ) => ReturnType<NonNullable<MCPOAuthErasureHooks['getServerConfig']>>;+ resolveAllowlists: (options: {+ userId: string;+ }) => ReturnType<MCPOAuthErasureHooks['resolveAllowlists']>;+ };+ getUserPrincipals: (params: {+ userId: string;+ }) => ReturnType<MCPOAuthErasureHooks['getUserPrincipals']>;+ uninstallOAuthMCP: MCPOAuthErasureHooks['uninstallOAuthMCP'];+ };+ /** Data-layer methods used by the cascade. */+ db: ErasureBaseDeps;+ getAppConfig: (options: {+ role?: string;+ userId: string;+ tenantId?: string;+ idOnTheSource?: string | null;+ failClosed?: boolean;+ }) => Promise<{ endpoints?: { agents?: { checkpointer?: TCheckpointerConfig } } } | undefined>;+ processDeleteRequest: (params: {+ req: Record<string, unknown>;+ files: ErasureFile[];+ }) => Promise<unknown>;+ invalidateCachedTools: (params: { userId: string; serverName: string }) => Promise<unknown>;+ getMCPManager: () =>+ | { disconnectUserConnection: (userId: string, serverName: string) => Promise<unknown> }+ | null+ | undefined;+ openCheckpointDeletion: UserErasureDeps['openCheckpointDeletion'];+ deleteSharedLinks: UserErasureDeps['deleteSharedLinks'];+}++/**+ * Builds the `eraseUserData` dependencies for an account deleted by someone else. Stored files+ * are removed with a request that carries the target user, since provider clients and+ * credentials are resolved from the request user, not the calling admin.+ */+export function createErasureResolver(deps: ErasureResolverDeps) {+ return async function resolveErasure({+ id,+ role,+ tenantId,+ idOnTheSource,+ profile,+ }: AdminErasureTarget): Promise<AdminErasureContext> {+ /** The calling admin's tenant context must not decide whose rows an id-keyed lookup reads. */+ const inTargetTenant = <T>(operation: () => Promise<T>): Promise<T> =>+ deps.withTenant(tenantId, operation);+ /** A failed override lookup must abort the deletion, not fall back to a base config that cannot reach the target's storage. */+ const appConfig = await inTargetTenant(async () =>+ deps.getAppConfig({+ role,+ userId: id,+ tenantId,+ idOnTheSource,+ failClosed: true,+ }),+ );+ const oauth = deps.mcpOAuth;+ const fileRequest = {+ user: { ...profile, id, _id: id, role, tenantId },+ /** Assistants client setup reads these Express request fields. */+ baseUrl: '',+ query: {},+ body: {},+ config: appConfig,+ };+ return {+ checkpointer: appConfig?.endpoints?.agents?.checkpointer,+ deps: {+ ...deps.db,+ openCheckpointDeletion: deps.openCheckpointDeletion,+ deleteSharedLinks: deps.deleteSharedLinks,+ deleteUserAgents: (userId) => inTargetTenant(async () => deps.db.deleteUserAgents(userId)),+ findMCPServerOAuthTokenUserIds: (serverName) =>+ inTargetTenant(async () => deps.db.findMCPServerOAuthTokenUserIds(serverName)),+ findMCPServerViewerEntries: (serverId) =>+ inTargetTenant(async () => deps.db.findMCPServerViewerEntries(serverId)),+ deleteStoredFiles: (files) =>+ inTargetTenant(async () => deps.processDeleteRequest({ req: fileRequest, files })),+ invalidateMCPServerTools: (serverName) =>+ deps.invalidateCachedTools({ userId: id, serverName }),+ disconnectMCPServer: async (serverName) => {+ await deps.getMCPManager()?.disconnectUserConnection(id, serverName);+ },+ mcpOAuth: oauth && {+ getServerConfig: (serverName) =>+ inTargetTenant(async () => oauth.getRegistry().getServerConfig(serverName, id)),+ getUserPrincipals: (candidateId) =>+ inTargetTenant(async () => oauth.getUserPrincipals({ userId: candidateId })),+ resolveAllowlists: (candidateId) =>+ inTargetTenant(async () =>+ oauth.getRegistry().resolveAllowlists({ userId: candidateId }),+ ),+ uninstallOAuthMCP: (candidateId, pluginKey, cleanupConfig, serverConfig) =>+ inTargetTenant(async () =>+ oauth.uninstallOAuthMCP(candidateId, pluginKey, cleanupConfig, serverConfig),+ ),+ invalidateUserTools: (candidateId, serverName) =>+ inTargetTenant(async () =>+ deps.invalidateCachedTools({ userId: candidateId, serverName }),+ ),+ disconnectUser: (candidateId, serverName) =>+ inTargetTenant(async () =>+ deps.getMCPManager()?.disconnectUserConnection(candidateId, serverName),+ ),+ },+ },+ };+ };+}